Talk at Rust meetup

.gitignore

@@ -5,3 +5,5 @@ requirements.txt
 /rust-tfrecord
 /.venv
 /target_maturin
+__pycache__/
+.DS_Store

README.md

@@ -10,7 +10,9 @@ Examples are loaded into native PyTorch `Tensor`s.
 
 The wheel can be installed on any Linux system with Python 3.8 or higher:
 
-    pip3 install rustfrecord
+```bash
+pip3 install rustfrecord
+````
 
 ## Getting Started
 
@@ -43,14 +45,11 @@ Repo: https://github.com/gavrie/rustfrecord
 
 To develop this package (not just use it), you need to install the Rust compiler and the Python development headers.
 
-    pip install uv
-    uv venv
-    source .venv/bin/activate
-
-    uv pip compile pyproject.toml -o requirements.txt
-    uv pip install -r requirements.txt
+```python
+pip install uv  # if needed
 
-    export LIBTORCH_USE_PYTORCH=1
-    CARGO_TARGET_DIR=target_maturin maturin develop
+export LIBTORCH_USE_PYTORCH=1
+CARGO_TARGET_DIR=target_maturin maturin develop
 
-    python main.py
+uv run pytest -sv test_rustfrecord.py
+```

pyproject.toml

@@ -24,3 +24,8 @@ build-backend = "maturin"
 
 [tool.maturin]
 features = ["pyo3/extension-module"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.5",
+]

talk.md

@@ -0,0 +1,123 @@
+---
+marp: true
+---
+
+# From TensorFlow to PyTorch
+## With some help from Rust
+
+<br>
+
+Gavrie Philipson
+Rusty Bits Software Ltd.
+June 2025
+
+---
+
+# About Me
+
+## Gavrie Philipson
+
+- Rust, Python, Cloud, Backend, DevOps, and more.
+- Bootstrapping software development teams: Training, mentoring, and hiring
+- Consulting to startup companies on software development and architecture
+
+<br>
+
+Rusty Bits Software Ltd.
+https://rustybits.io
+gavrie@rustybits.io
+
+---
+
+# About You
+
+---
+
+# Using Rust to improve Python
+
+<br>
+<br>
+<br>
+<br>
+<br>
+
+![bg 40%](images/python-logo.png)
+![bg 60%](images/ferris.png)
+![bg 50%](images/iron-oxide.jpg)
+
+<br>
+<br>
+
+[Astral](https://astral.sh)
+[PyO<sub>3</sub>](https://github.com/PyO3)
+
+---
+
+# The Mission
+
+- Port ML model from TensorFlow to PyTorch
+- Lots of training data in `TFRecord` format
+
+---
+
+# The `TFRecord` format
+
+- A sequence of `HashMap<String, Vec<T>>`
+- `where T: u8 | f32 | i64`
+- Serialized with `protobuf`
+
+---
+
+# `TFRecord` Example
+
+```python
+[
+  {
+    "label": "cat",
+    "image/shape": [320, 200, 3],
+    "image/encoded": [0x12, 0x34, 0x56, ...],
+  },
+  {
+    "label": "dog",
+    "image/shape": [320, 200, 3],
+    "image/encoded": [0x78, 0x9a, 0xbc, ...],
+  },
+]
+```
+
+---
+
+# The Constraints
+
+- Dependencies (look at venv size)
+- Performance: Keep GPUs busy
+- Ease of use for Python devs
+
+---
+
+# Challenge: Getting Test Data
+
+- No access to the original data
+- Vibe code [some Python](tf_example/main.py) to generate test data!
+
+---
+
+# Playing on Rust's strengths
+
+- Designing with types
+- Dive into the [Rust implementation](tfrecord_reader/src/lib.rs) and [`tests.rs`](tfrecord_reader/src/tests.rs)
+
+---
+
+# The End Result
+
+- `pip install rustfrecord` 
+- [`test_rustfrecord.py`](test_rustfrecord.py)
+- [`src/lib.rs`](src/lib.rs)
+
+---
+
+# Getting the Code
+
+https://pypi.org/project/rustfrecord/
+https://github.com/gavrie/rustfrecord

test_rustfrecord.py

@@ -5,6 +5,8 @@
 # torch.set_num_threads(1)
 # torch.set_num_interop_threads(1)
 
+filename = "tf_example/sample_images.tfrecord"
+
 
 class TFRecordDataset(torch.utils.data.IterableDataset):
     def __init__(self, filename: str, compressed: bool = True, features: list = None):
@@ -23,7 +25,6 @@ def __iter__(self):
 
 
 def test_loader():
-    filename = "data/002scattered.training_examples.tfrecord"
     ds = TFRecordDataset(
         filename,
         compressed=filename.endswith(".gz"),
@@ -52,9 +53,7 @@ def test_loader():
 
 
 def test_dataset():
-    filename = "data/002scattered.training_examples.tfrecord"
-
-    for _ in range(10):
+    for _ in range(1):
         ds = TFRecordDataset(
             filename,
             compressed=filename.endswith(".gz"),
@@ -68,17 +67,14 @@ def test_dataset():
         print()
 
         for i, features in enumerate(ds):
-            label: Tensor = features["label"]
+            label: Tensor = features["label"].tobytes().decode("utf-8")
             shape = torch.Size(tuple(features["image/shape"]))
-            image: Tensor = features["image/encoded"][0].reshape(shape)
+            image: Tensor = features["image/encoded"].reshape(shape)
 
-            if i % 1000 == 0:
-                print(i, label, image.shape)
+            print(i, label, image.shape)
 
 
 def test_reader():
-    filename = "data/002scattered.training_examples.tfrecord"
-
     r = Reader(
         filename,
         compressed=filename.endswith(".gz"),
@@ -94,7 +90,7 @@ def test_reader():
     for i, features in enumerate(r):
         label: Tensor = features["label"]
         shape = torch.Size(tuple(features["image/shape"]))
-        image: Tensor = features["image/encoded"][0].reshape(shape)
+        image: Tensor = features["image/encoded"].reshape(shape)
 
         if i % 1000 == 0:
             print(i, label, image.shape)

tf_example/.gitignore

@@ -0,0 +1,2 @@
+sample_images.tfrecord
+generated_images/