Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 31 additions & 4 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,29 @@ on:
branches:
- main

env:
REDIS_VERSION: 8.4.0

jobs:
linux:
check-style:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: pnpm/action-setup@v4
with:
version: 9
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- run: pip install -e .[dev]
- run: scripts/check-prettier.sh
- run: ruff check .
- run: cargo fmt --all -- --check
- run: cargo clippy --workspace -- -D warnings

tests-linux:
runs-on: ubuntu-latest
strategy:
matrix:
Expand All @@ -31,8 +52,14 @@ jobs:

- run: pip install -e .[tests]

- run: scripts/check-prettier.sh
- run: cargo fmt --all -- --check
- run: cargo clippy --workspace -- -D warnings
- run: pytest tests/
- run: cargo test --workspace --locked

tests-pass:
needs:
- check-style
- tests-linux
if: (github.repository_owner == 'ccxlv')
runs-on: ubuntu-latest
steps:
- run: echo "All tests passed ✅"
4 changes: 2 additions & 2 deletions crates/fluxqueue-worker/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ async fn main() -> Result<()> {
fluxqueue_worker::run_worker(
shutdown_rx,
concurrency,
&redis_url,
redis_url,
tasks_module_path,
&queue,
queue,
save_dead_tasks,
)
.await
Expand Down
23 changes: 13 additions & 10 deletions crates/fluxqueue-worker/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,27 @@ use fluxqueue_common::{Task, deserialize_raw_task_data};
pub async fn run_worker(
mut shutdown: watch::Receiver<bool>,
concurrency: usize,
redis_url: &str,
redis_url: String,
tasks_module_path: String,
queue_name: &str,
queue_name: String,
save_dead_tasks: bool,
) -> Result<()> {
let redis_client = RedisClient::new(redis_url).await.map_err(|e| {
let redis_client = RedisClient::new(&redis_url).await.map_err(|e| {
tracing::error!("{}", e);
std::process::exit(1);
})?;
let redis_client = Arc::new(redis_client);

let task_functions = get_task_functions(&tasks_module_path, queue_name).map_err(|e| {
let task_functions = get_task_functions(&tasks_module_path, &queue_name).map_err(|e| {
tracing::error!("{}", e);
std::process::exit(1);
})?;
let task_names: Vec<&String> = task_functions.iter().map(|(name, _obj)| name).collect();

initial_logs(
queue_name,
&queue_name,
concurrency,
redis_url,
&redis_url,
&tasks_module_path,
&task_names,
);
Expand Down Expand Up @@ -630,9 +630,8 @@ mod tests {

let module_path_str = get_test_module_path("test_tasks_module.py");
let redis_version = std::env::var("REDIS_VERSION").unwrap_or("latest".to_string());
let redis_url = "redis://localhost:6379";

GenericImage::new("redis", &redis_version)
let container = GenericImage::new("redis", &redis_version)
.with_exposed_port(6379.tcp())
.with_wait_for(WaitFor::message_on_stdout("Ready to accept connections"))
.with_network("bridge")
Expand All @@ -641,19 +640,23 @@ mod tests {
.await
.expect("Failed to start Redis");

let mapped_port = container.get_host_port_ipv4(6379).await?;
let redis_url = format!("redis://localhost:{:?}", mapped_port);
let cloned_redis_url = redis_url.clone();

let (shutdown_tx, shutdown_rx) = watch::channel(false);

let worker_handle = tokio::spawn(run_worker(
shutdown_rx,
4,
redis_url,
module_path_str,
"default",
"default".to_string(),
false,
));

sleep(Duration::from_secs(5)).await;
enqueue_tasks(redis_url).await?;
enqueue_tasks(&cloned_redis_url).await?;
sleep(Duration::from_secs(5)).await;

let _ = shutdown_tx.send(true);
Expand Down
11 changes: 1 addition & 10 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,7 @@ Repository = "https://github.com/CCXLV/fluxqueue"
Documentation = "https://fluxqueue.ccxlv.dev"

[project.optional-dependencies]
dev = [
"ruff",
"pytest-asyncio",
"python-dotenv",
"redis",
"mkdocs-material",
"mkdocstrings",
"mkdocstrings-python",
"maturin",
]
dev = ["ruff"]
tests = [
"pytest-asyncio",
"python-dotenv",
Expand Down