Add Python support.

2026-04-25 17:10:41 +01:00
parent 5ff4b3ee5c
commit 48adf25837
11 changed files with 467 additions and 14 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1535,6 +1535,12 @@ version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"

+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.9"
@@ -1860,6 +1866,15 @@ dependencies = [
 "hashbrown 0.17.0",
 ]

+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "inout"
 version = "0.1.4"
@@ -2041,6 +2056,15 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"

+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -2336,6 +2360,12 @@ dependencies = [
 "windows-sys 0.61.2",
 ]

+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
 [[package]]
 name = "potential_utf"
 version = "0.1.5"
@@ -2369,6 +2399,69 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "pyo3"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -3013,6 +3106,7 @@ dependencies = [
 "aws-sdk-s3",
 "futures-util",
 "lapin",
+ "pyo3",
 "reqwest",
 "serde",
 "serde_json",
@@ -3125,6 +3219,12 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
 [[package]]
 name = "tcp-stream"
 version = "0.28.0"
@@ -3406,6 +3506,12 @@ version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,6 +5,13 @@ edition = "2021"
 description = "Opinionated Rust framework for queue-driven microservices"
 license = "MIT"

+[lib]
+crate-type = ["rlib", "cdylib"]
+
+[features]
+default = []
+python = ["dep:pyo3"]
+
 [dependencies]
 aws-config = "1"
 aws-sdk-s3 = "1"
@@ -16,6 +23,7 @@ serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+pyo3 = { version = "0.23", optional = true, features = ["extension-module", "abi3-py38"] }

 [dev-dependencies]
 anyhow = "1"
--- a/README.md
+++ b/README.md
@@ -44,6 +44,48 @@ Then fetch and build dependencies:
 cargo build
 ```

+## Python Usage
+
+This crate also exposes Python bindings behind the Cargo feature `python`.
+The Python callback receives a request ID plus callable `read_file` and
+`write_file` helpers and should yield tuples of `(result_id, case_variable)`.
+
+```python
+from typing import Generator
+
+from slingshot_microservice.typing import ReadFileFn, WriteFileFn
+from slingshot_microservice import Microservice
+
+
+def process(
+    request: int,
+    read_file: ReadFileFn,
+    write_file: WriteFileFn,
+) -> Generator[tuple[int, bool | int | str], None, None]:
+    reader = read_file("in", request)
+    input_data = reader.read().decode()
+
+    writer = write_file("out", request)
+    writer.write(f"Hello {input_data}".encode())
+
+    yield (request, True)
+
+
+microservice = Microservice("simple-py-microservice", "sys-map.slingshot.cv", process)
+microservice.start()
+```
+
+### Building The Python Extension
+
+Build with:
+
+```bash
+cargo build --release --features python
+```
+
+The generated shared library can then be imported by Python as
+`slingshot_microservice`.
+
 ## Example Usage

 ```rust
--- a/examples/py_simple.py
+++ b/examples/py_simple.py
@@ -0,0 +1,18 @@
+from slingshot_microservice.typing import ReadFileFn, WriteFileFn
+from slingshot_microservice import Microservice
+from typing import Generator
+
+def process(
+	request: int,
+	read_file: ReadFileFn,
+	write_file: WriteFileFn,
+) -> Generator[tuple[int, bool | int | str], None, None]:
+	reader = read_file("in", request)
+	input_data = reader.read().decode()
+	writer = write_file("out", request)
+	writer.write(f"Hello {input_data}".encode())
+	yield (request, True)
+
+
+microservice = Microservice("simple-py-microservice", "sys-map.slingshot.cv", process)
+microservice.start()
--- a/slingshot_microservice/init.py
+++ b/slingshot_microservice/init.py
@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import importlib.util
+import pathlib
+import sys
+from types import ModuleType
+
+
+def _load_native() -> ModuleType:
+    package_dir = pathlib.Path(__file__).resolve().parent
+    project_root = package_dir.parent
+
+    # Common library locations for local cargo builds and wheel installs.
+    candidates = [
+        project_root / "target" / "debug" / "libslingshot_microservice.so",
+        project_root / "target" / "release" / "libslingshot_microservice.so",
+    ]
+    candidates.extend(package_dir.glob("_native*.so"))
+    candidates.extend(package_dir.glob("libslingshot_microservice*.so"))
+
+    module_name = "slingshot_microservice._native"
+    for candidate in candidates:
+        if not candidate.exists():
+            continue
+
+        spec = importlib.util.spec_from_file_location(module_name, candidate)
+        if spec is None or spec.loader is None:
+            continue
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        return module
+
+    raise ModuleNotFoundError(
+        "Native extension is not built. Build it with: cargo build --features python"
+    )
+
+
+_native = _load_native()
+Microservice = _native.Microservice
+
+__all__ = ["Microservice"]
--- a/slingshot_microservice/pycache/init.cpython-312.pyc
+++ b/slingshot_microservice/pycache/init.cpython-312.pyc
--- a/slingshot_microservice/pycache/init.cpython-314.pyc
+++ b/slingshot_microservice/pycache/init.cpython-314.pyc
--- a/slingshot_microservice/pycache/typing.cpython-312.pyc
+++ b/slingshot_microservice/pycache/typing.cpython-312.pyc
--- a/slingshot_microservice/pycache/typing.cpython-314.pyc
+++ b/slingshot_microservice/pycache/typing.cpython-314.pyc
--- a/slingshot_microservice/typing.py
+++ b/slingshot_microservice/typing.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import BinaryIO, Generator, Protocol, TypeAlias
+
+CaseVariable: TypeAlias = bool | int | str
+
+
+class ReadFileFn(Protocol):
+    def __call__(self, key: str, id: int) -> BinaryIO: ...
+
+
+class WriteFileFn(Protocol):
+    def __call__(self, key: str, id: int) -> BinaryIO: ...
+
+
+class ProcessFn(Protocol):
+    def __call__(
+        self,
+        request: int,
+        read_file: ReadFileFn,
+        write_file: WriteFileFn,
+    ) -> Generator[tuple[int, CaseVariable], None, None]: ...
+
+
+__all__ = [
+    "CaseVariable",
+    "ReadFileFn",
+    "WriteFileFn",
+    "ProcessFn",
+]
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,6 +2,8 @@ use std::collections::HashMap;
 use std::error::Error;
 use std::fs::{self, File};
 use std::io::{Cursor, ErrorKind, Read};
+#[cfg(feature = "python")]
+use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::sync::Arc;
@@ -26,12 +28,20 @@ use serde_json::Value;
 use tokio::io::AsyncReadExt;
 use tracing_subscriber::EnvFilter;

+#[cfg(feature = "python")]
+use pyo3::exceptions::{PyRuntimeError, PyTypeError};
+#[cfg(feature = "python")]
+use pyo3::prelude::*;
+#[cfg(feature = "python")]
+use pyo3::types::{PyAny, PyBool, PyBytes, PyInt, PyModule, PyString};
+
 pub type AnyError = Box<dyn Error + Send + Sync + 'static>;

 pub type ReadFile = Box<dyn Read + Send + 'static>;
 pub type ReadFileFn = dyn Fn(&str, u64) -> Result<ReadFile, AnyError> + Send + Sync + 'static;
 pub type WriteFileFn = dyn Fn(&str, u64) -> Result<File, AnyError> + Send + Sync + 'static;
-type ProcessFn = dyn Fn(u64, &ReadFileFn, &WriteFileFn) -> Result<Vec<(u64, CaseKey)>, AnyError>
+
+type ProcessFn = dyn Fn(u64, Arc<ReadFileFn>, Arc<WriteFileFn>) -> Result<Vec<(u64, CaseKey)>, AnyError>
 	+ Send
 	+ Sync
 	+ 'static;
@@ -79,6 +89,19 @@ pub struct Microservice {
 }

 impl Microservice {
+	fn new_case_key(
+		name: impl Into<String>,
+		config_host: impl Into<String>,
+		process: Arc<ProcessFn>,
+	) -> Self {
+		init_tracing();
+		Self {
+			name: name.into(),
+			config_host: config_host.into(),
+			process,
+		}
+	}
+
 	/// Create a new microservice runtime.
 	///
 	/// `process` accepts an inbound request ID, a `read_file` function, and a
@@ -93,10 +116,10 @@ impl Microservice {
 		init_tracing();
 		let process_wrapper = move |
 			request: u64,
-			read_file: &ReadFileFn,
-			write_file: &WriteFileFn,
+			read_file: Arc<ReadFileFn>,
+			write_file: Arc<WriteFileFn>,
 		| -> Result<Vec<(u64, CaseKey)>, AnyError> {
-			let outputs = process(request, read_file, write_file)?;
+			let outputs = process(request, read_file.as_ref(), write_file.as_ref())?;
 			let mut mapped = Vec::with_capacity(outputs.len());
 			for (id, case) in outputs {
 				let value = serde_json::to_value(case)
@@ -190,7 +213,7 @@ impl Microservice {
 			let read_config_host = config_host.clone();
 			let read_microservice_name = microservice_name.clone();

-			let read_file = move |key: &str, id: u64| -> Result<ReadFile, AnyError> {
+			let read_file: Arc<ReadFileFn> = Arc::new(move |key: &str, id: u64| -> Result<ReadFile, AnyError> {
 				let bucket = resolve_bucket_name(
 					&read_config_host,
 					&read_microservice_name,
@@ -202,9 +225,9 @@ impl Microservice {
 					.lock()
 					.map_err(|e| format!("file context lock poisoned for read_file: {}", e))?;
 				guard.read_file(s3_read_client.as_ref(), &bucket, id)
-			};
+			});

-			let write_file = move |key: &str, id: u64| -> Result<File, AnyError> {
+			let write_file: Arc<WriteFileFn> = Arc::new(move |key: &str, id: u64| -> Result<File, AnyError> {
 				let bucket = resolve_bucket_name(
 					&config_host,
 					&microservice_name,
@@ -215,9 +238,9 @@ impl Microservice {
 					.lock()
 					.map_err(|e| format!("file context lock poisoned for write_file: {}", e))?;
 				guard.write_file(&bucket, id)
-			};
+			});

-			let outputs = (self.process)(request_id, &read_file, &write_file)?;
+			let outputs = (self.process)(request_id, Arc::clone(&read_file), Arc::clone(&write_file))?;
 			{
 				let mut guard = file_context
 					.lock()
@@ -408,9 +431,9 @@ fn fetch_rabbitmq_url_from_sys_map() -> Result<String, AnyError> {
 }

 async fn fetch_s3_client_from_sys_map() -> Result<Arc<Client>, AnyError> {
-	let response = reqwest::blocking::get("https://sys-map.slingshot.cv/object-storage")?;
+	let response = reqwest::get("https://sys-map.slingshot.cv/object-storage").await?;
 	let response = response.error_for_status()?;
-	let config = response.json::<ObjectStorageConfig>()?;
+	let config = response.json::<ObjectStorageConfig>().await?;

 	let host = single_value(&config.host, "host")?;
 	let access_key_ref = single_value(&config.pass_access_key, "pass:access-key")?;
@@ -456,9 +479,13 @@ fn resolve_bucket_name(
 	}

 	let url = bucket_mapping_url(config_host, microservice_name, key);
-	let response = reqwest::blocking::get(&url)?;
-	let response = response.error_for_status()?;
-	let bucket_name = response.text()?.trim().to_string();
+	let bucket_name = tokio::task::block_in_place(|| {
+		tokio::runtime::Handle::current().block_on(async {
+			let response = reqwest::get(&url).await?;
+			let response = response.error_for_status()?;
+			Ok::<String, AnyError>(response.text().await?.trim().to_string())
+		})
+	})?;

 	if bucket_name.is_empty() {
 		return Err(format!("bucket mapping '{}' returned an empty bucket name", url).into());
@@ -614,3 +641,182 @@ async fn publish_outputs(

 	Ok(())
 }
+
+#[cfg(feature = "python")]
+fn any_error_to_py(err: AnyError) -> PyErr {
+	PyRuntimeError::new_err(err.to_string())
+}
+
+#[cfg(feature = "python")]
+fn case_key_from_py_value(value: &Bound<'_, PyAny>) -> PyResult<CaseKey> {
+	if value.is_instance_of::<PyBool>() {
+		return Ok(CaseKey::Bool(value.extract::<bool>()?));
+	}
+
+	if value.is_instance_of::<PyInt>() {
+		return Ok(CaseKey::Int(value.extract::<i128>()?));
+	}
+
+	if value.is_instance_of::<PyString>() {
+		return Ok(CaseKey::String(value.extract::<String>()?));
+	}
+
+	Err(PyTypeError::new_err(
+		"case variable must be one of: bool, int, string",
+	))
+}
+
+#[cfg(feature = "python")]
+#[pyclass(name = "ReadFileFn")]
+struct PyReadFileFn {
+	inner: Arc<ReadFileFn>,
+}
+
+#[cfg(feature = "python")]
+#[pymethods]
+impl PyReadFileFn {
+	fn __call__(&self, py: Python<'_>, key: &str, id: u64) -> PyResult<Py<PyAny>> {
+		let mut reader = (self.inner)(key, id).map_err(any_error_to_py)?;
+		let mut data = Vec::new();
+		reader.read_to_end(&mut data).map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+
+		let io = py.import("io")?;
+		let bytes_io = io
+			.getattr("BytesIO")?
+			.call1((PyBytes::new(py, &data),))?;
+
+		Ok(bytes_io.unbind())
+	}
+}
+
+#[cfg(feature = "python")]
+#[pyclass]
+struct PyWriteHandle {
+	file: Arc<Mutex<File>>,
+}
+
+#[cfg(feature = "python")]
+#[pymethods]
+impl PyWriteHandle {
+	fn write(&self, data: &[u8]) -> PyResult<usize> {
+		let mut file = self
+			.file
+			.lock()
+			.map_err(|e| PyRuntimeError::new_err(format!("write lock poisoned: {}", e)))?;
+		file.write_all(data)
+			.map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+		Ok(data.len())
+	}
+
+	fn flush(&self) -> PyResult<()> {
+		let mut file = self
+			.file
+			.lock()
+			.map_err(|e| PyRuntimeError::new_err(format!("flush lock poisoned: {}", e)))?;
+		file.flush()
+			.map_err(|e| PyRuntimeError::new_err(e.to_string()))
+	}
+}
+
+#[cfg(feature = "python")]
+#[pyclass(name = "WriteFileFn")]
+struct PyWriteFileFn {
+	inner: Arc<WriteFileFn>,
+}
+
+#[cfg(feature = "python")]
+#[pymethods]
+impl PyWriteFileFn {
+	fn __call__(&self, py: Python<'_>, key: &str, id: u64) -> PyResult<Py<PyWriteHandle>> {
+		let file = (self.inner)(key, id).map_err(any_error_to_py)?;
+		Py::new(
+			py,
+			PyWriteHandle {
+				file: Arc::new(Mutex::new(file)),
+			},
+		)
+	}
+}
+
+#[cfg(feature = "python")]
+fn run_python_process(
+	process: &Py<PyAny>,
+	request: u64,
+	read_file: Arc<ReadFileFn>,
+	write_file: Arc<WriteFileFn>,
+) -> Result<Vec<(u64, CaseKey)>, AnyError> {
+	Python::with_gil(|py| -> Result<Vec<(u64, CaseKey)>, AnyError> {
+		let py_read = Py::new(py, PyReadFileFn { inner: read_file })
+			.map_err(|e| format!("failed to build Python ReadFileFn wrapper: {}", e))?;
+		let py_write = Py::new(py, PyWriteFileFn { inner: write_file })
+			.map_err(|e| format!("failed to build Python WriteFileFn wrapper: {}", e))?;
+
+		let returned = process
+			.call1(py, (request, py_read, py_write))
+			.map_err(|e| format!("Python process callback failed: {}", e))?;
+
+		let iter = returned
+			.bind(py)
+			.try_iter()
+			.map_err(|e| format!("process return value must be iterable: {}", e))?;
+
+		let mut outputs = Vec::new();
+		for item in iter {
+			let item = item.map_err(|e| format!("failed to iterate process outputs: {}", e))?;
+			let (id, case_obj): (u64, Py<PyAny>) = item
+				.extract()
+				.map_err(|e| format!("each output must be a tuple (int, case): {}", e))?;
+			let case = case_key_from_py_value(case_obj.bind(py))
+				.map_err(|e| format!("invalid case variable: {}", e))?;
+			outputs.push((id, case));
+		}
+
+		Ok(outputs)
+	})
+}
+
+#[cfg(feature = "python")]
+#[pyclass(name = "Microservice")]
+struct PyMicroservice {
+	name: String,
+	config_host: String,
+	process: Py<PyAny>,
+}
+
+#[cfg(feature = "python")]
+#[pymethods]
+impl PyMicroservice {
+	#[new]
+	fn new(name: String, config_host: String, process: Py<PyAny>) -> PyResult<Self> {
+		Python::with_gil(|py| {
+			if !process.bind(py).is_callable() {
+				return Err(PyTypeError::new_err("process must be callable"));
+			}
+
+			Ok(Self {
+				name,
+				config_host,
+				process,
+			})
+		})
+	}
+
+	fn start(&self) -> PyResult<()> {
+		let process = Python::with_gil(|py| self.process.clone_ref(py));
+		let microservice = Microservice::new_case_key(
+			self.name.clone(),
+			self.config_host.clone(),
+			Arc::new(move |request, read_file, write_file| run_python_process(&process, request, read_file, write_file)),
+		);
+
+		microservice.start().map_err(any_error_to_py)
+	}
+}
+
+#[cfg(feature = "python")]
+#[pymodule]
+fn _native(_py: Python<'_>, module: &Bound<'_, PyModule>) -> PyResult<()> {
+	module.add_class::<PyMicroservice>()?;
+
+	Ok(())
+}