diff --git a/.gitignore b/.gitignore index ee24b33..2728791 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,8 @@ expo-env.d.ts npm-debug.* yarn-debug.* yarn-error.* +metro.log +metro-error.log # macOS .DS_Store @@ -49,6 +51,7 @@ yarn-error.* # AI model binaries require independently verified redistribution rights. /assets/models/*.tflite +!/assets/models/efficientdet-lite0-int8-v1.tflite /.local-models/ # typescript diff --git a/App.tsx b/App.tsx index 2a6708a..420ac2e 100644 --- a/App.tsx +++ b/App.tsx @@ -71,10 +71,15 @@ function VehicleAnalysisSheet({ session, onClose, onRequestAnalysis }: { session {analysis.status === 'completed' ? 'Lokalna analiza je dovršena' : 'Lokalna analiza nije dostupna'} {analysis.status === 'ready-for-model' - ? 'Model detekcije nije uključen u ovu javnu verziju. Snimka ostaje spremljena i spremna za buduću lokalnu analizu.' + ? 'Lokalni model detekcije nije se mogao pokrenuti u ovom buildu. Snimka ostaje spremljena; tehnički detalj prikazan je u ograničenjima izvještaja.' : 'Analiza koristi lokalni model vozila i OCR. Rezultati oznake ostaju kandidati dok se ne potvrde kroz više kadrova.'} Dokazni kadrovi: {analysis.evidenceFrames.length} · tragovi vozila: {analysis.vehicleTracks.length} + {analysis.audioSummary && ( + + Zvuk videa: prosjek {formatDbfs(analysis.audioSummary.averageDbfs)} · vrh {formatDbfs(analysis.audioSummary.peakDbfs)} · {analysis.audioSummary.sampleCount} očitanja + + )} {analysis.vehicleTracks.map(track => ( {track.evidenceCropUri @@ -175,7 +180,10 @@ function ReportSheet({ session, onClose, onExport }: { session: Session; onClose Trajanje: {formatDuration(session.durationSeconds)} Lokacija: {formatLocation(session.location)} - Buka: prosjek {formatDbfs(session.noiseAverageDbfs)} · vrh {formatDbfs(session.noisePeakDbfs)} + + Zvuk: prosjek {formatDbfs(report?.audioSummary?.averageDbfs ?? session.noiseAverageDbfs)} · vrh {formatDbfs(report?.audioSummary?.peakDbfs ?? session.noisePeakDbfs)} + + {report?.audioSummary && Dekodirano: {report.audioSummary.sampleCount} očitanja po {report.audioSummary.windowMs} ms} {analysisLabel(session.analysis)} {report ? ( @@ -188,6 +196,7 @@ function ReportSheet({ session, onClose, onExport }: { session: Session; onClose {track.plateCandidates.some(candidate => candidate.confirmationCount >= 2) ? track.plateCandidates.filter(candidate => candidate.confirmationCount >= 2).map(candidate => ( {candidate.normalizedText} · {candidate.confirmationCount} kadar(a) · {candidate.confidenceLevel} )) : Nema potvrđenog kandidata oznake.} + {track.noise && Zvuk uz prolazak: prosjek {formatDbfs(track.noise.averageDbfs)} · vrh {formatDbfs(track.noise.peakDbfs)}} )) : U dokaznim kadrovima nisu pronađena vozila.} {report.unassignedPlateCandidates.length > 0 && ( diff --git a/README.md b/README.md index 06e574a..bed45aa 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Mobilni istraživačko-razvojni projekt za snimanje i kasniju analizu prometnih scena pomoću kamere, mikrofona i umjetne inteligencije na pametnom telefonu. -> Trenutačna verzija: **v0.2.0 razvojna**. Aplikacija lokalno snima videosesije sa zvukom i priprema dokazne kadrove za eksperimentalnu analizu. Javni izvor ne distribuira AI model. Aplikacija ne mjeri niti potvrđuje stvarnu brzinu vozila ili razinu buke. +> Trenutačna verzija: **v0.2.0 razvojna**. Aplikacija lokalno snima videosesije sa zvukom, izdvaja dokazne kadrove i koristi licencirani lokalni EfficientDet-Lite0 model za eksperimentalnu detekciju vozila. Aplikacija ne mjeri niti potvrđuje stvarnu brzinu vozila ili razinu buke. ## Cilj @@ -46,7 +46,7 @@ flowchart TB - Lokalni dnevnik sesija s vremenom snimanja, trajanjem i veličinom datoteke. - Pregled snimljenog videa unutar aplikacije i trajno brisanje odabrane sesije. - Sučelje prilagođeno radu na terenu: status, mjerač vremena i brzo zaustavljanje snimanja. -- Eksperimentalna lokalna obrada: izdvajanje i rangiranje kadrova, osnovno praćenje, OCR kandidati oznaka i korelacija audio uzoraka. Detektor vozila zahtijeva zaseban pravilno licenciran model. +- Eksperimentalna lokalna obrada: EfficientDet-Lite0 detekcija vozila, izdvajanje i rangiranje kadrova, osnovno praćenje, OCR kandidati oznaka te dekodiranje i vremenska korelacija zvuka iz uvezenog videa. - Razvojna dijagnostika prikazuje confidence, vrijeme i okvir svake detekcije po tragu vozila. - Automatizirane provjere obuhvaćaju lint, TypeScript i testove čistih analitičkih modula. @@ -81,7 +81,7 @@ Pokrenite razvojni poslužitelj i otvorite ga development buildom. Za rad kamere ## Android development build i terensko testiranje -Za lokalni AI model aplikacija će koristiti vlastiti Android development build, a ne Expo Go. Konfiguracija je pripremljena u `eas.json` pod profilom `development` i proizvodi interni `.apk` paket. +Za lokalni AI model aplikacija koristi vlastiti Android development build, a ne Expo Go. Konfiguracija je pripremljena u `eas.json` pod profilom `development` i proizvodi interni `.apk` paket. Nakon prijave u Expo račun, build se stvara naredbom: @@ -107,21 +107,22 @@ Prije početka snimanja potvrdite dozvole za kameru, mikrofon i lokaciju. Snimit ## Tehnologije -- Expo SDK 54 -- React Native 0.81.5 +- Expo SDK 57 +- React Native 0.86.3 - TypeScript - `expo-camera` - `expo-file-system` - `expo-video` +- `react-native-audio-api` - AsyncStorage ## AI model -Javni repozitorij ne uključuje binarni AI model. Zašto je prethodni razvojni artefakt uklonjen i koji su uvjeti za doprinos novog modela opisano je u [modelskoj kartici](docs/MODEL.md). +Javni repozitorij uključuje službeni EfficientDet-Lite0 int8 v1 pod Apache-2.0 licencom. Izvor, SHA-256, ulazno-izlazni ugovor i ograničenja opisani su u [modelskoj kartici](docs/MODEL.md). ## Smjer projekta -JEKA AOPS je otvoreni istraživačko-razvojni projekt pod licencom Apache-2.0. Sljedeći korak je integracija provjerljivo licenciranog lokalnog modela i terenska provjera u Android development buildu. Bez kalibracije scene rezultati se ne smiju predstavljati kao mjerenje. +JEKA AOPS je otvoreni istraživačko-razvojni projekt pod licencom Apache-2.0. Sljedeći korak je terenska provjera lokalnog modela, OCR-a i praćenja u Android development buildu. Bez kalibracije scene rezultati se ne smiju predstavljati kao mjerenje. ## Doprinos i sigurnost diff --git a/app.json b/app.json index 9576b5a..48996d3 100644 --- a/app.json +++ b/app.json @@ -27,7 +27,9 @@ ], "expo-asset", "expo-sharing", - "expo-status-bar" + "expo-status-bar", + "react-native-fast-tflite", + "react-native-audio-api" ], "ios": { "supportsTablet": true diff --git a/assets/models/efficientdet-lite0-int8-v1.tflite b/assets/models/efficientdet-lite0-int8-v1.tflite new file mode 100644 index 0000000..b43cc06 Binary files /dev/null and b/assets/models/efficientdet-lite0-int8-v1.tflite differ diff --git a/docs/MODEL.md b/docs/MODEL.md index 0690b93..83c9197 100644 --- a/docs/MODEL.md +++ b/docs/MODEL.md @@ -1,34 +1,48 @@ # Model detekcije vozila -## Status javne distribucije +JEKA AOPS uključuje lokalni **EfficientDet-Lite0 int8, verzija 1** za početnu detekciju cestovnih vozila. Model radi na uređaju; snimke i kadrovi ne šalju se vanjskom AI servisu. -Javni repozitorij namjerno **ne sadrži binarni AI model**. Raniji razvojni artefakt identificirao se kao EfficientDet Lite0 V1, ali nije postojao izvorni zapis preuzimanja ni upstream checksum kojim bi se nedvojbeno potvrdili porijeklo i pravo redistribucije. Zbog toga je uklonjen prije javne objave. +## Podrijetlo i licenca -`detectVehiclesInFrames` trenutačno je stabilna neutralna implementacija koja vraća prazan skup detekcija. Snimanje, lokalna pohrana, audio uzorkovanje, izdvajanje kadrova i izvještaji mogu se razvijati neovisno o odabiru modela. +- Izdavač: Google / TensorFlow +- Arhitektura: EfficientDet-Lite0 +- Skup podataka: COCO 2017, 80 klasa +- Fiksni izvor: +- Službena kartica: +- Službeni vodič: +- Licenca modela: Apache License 2.0 +- Datum preuzimanja: 2026-09-01 +- Veličina: 4.602.795 bajtova +- SHA-256: `0720bf247bd76e6594ea28fa9c6f7c5242be774818997dbbeffc4da460c723bb` -## Uvjeti za novi model +Model je spremljen kao `assets/models/efficientdet-lite0-int8-v1.tflite`. Runtime `react-native-fast-tflite` distribuira se pod MIT licencom. -Prije dodavanja ili automatskog preuzimanja modela pull request mora sadržavati: +## Ulaz i izlaz -1. naziv modela, izdavača, verziju, izvorni URL i datum preuzimanja; -2. SHA-256 preuzete datoteke i reproducibilan postupak provjere; -3. licencu modela, obavezne obavijesti i licencu skupa podataka; -4. dopušta li licenca redistribuciju binarnog artefakta i komercijalnu uporabu; -5. ulazni tip, oblik, RGB raspored i normalizaciju; -6. redoslijed, tipove i oblike izlaznih tenzora; -7. mapu klasa, confidence/NMS pragove i poznata ograničenja; -8. testove na sintetičkim ili pravilno licenciranim snimkama bez osobnih podataka. +- Ulaz: RGB `uint8`, `[1, 320, 320, 3]` +- Predobrada: očuvanje omjera slike uz letterbox rubove vrijednosti 114 +- Izlazi: okviri `[top, left, bottom, right]`, COCO klase, confidence i broj detekcija +- Prag: 0,32 +- Prihvaćene klase: bicycle, car, motorcycle, bus i truck +- Koordinate: nakon izvođenja vraćaju se u normalizirane koordinate izvornog kadra -## Integracijska granica +Kadrovi se izdvajaju lokalno iz videozapisa. Model se izvršava CPU delegatom radi predvidljive kompatibilnosti; kasnije se može zasebno provjeriti GPU delegat. -Adapter se implementira u `src/analysis/vehicleDetectionModel.ts` i mora zadržati potpis: +## OCR fallback -```ts -detectVehiclesInFrames(frames: EvidenceFrame[]): Promise -``` +Ako detektor ne pronađe vozilo, aplikacija ipak zadržava do tri najkvalitetnija cijela kadra i na njima pokreće OCR. Takvi kandidati nisu prostorno pridruženi vozilu i moraju biti jasno označeni kao nepouzdani. Registracijska oznaka ne prikazuje se kao potvrđena bez ponavljanja kroz više različitih kadrova. -Native biblioteka i config plugin dodaju se tek kada je odabrani model odobren. Model ne treba commitati ako se može reproducibilno preuzeti tijekom lokalne pripreme ili builda uz provjeru očekivanog SHA-256. +## Poznata ograničenja -## Ograničenja rezultata +- COCO model prepoznaje opće klase vozila; ne prepoznaje marku, model, identitet ni prometni prekršaj. +- Mala, zamućena, zaklonjena i noćna vozila mogu biti propuštena. +- Confidence nije dokaz točnosti niti certificirana mjera. +- OCR može zamijeniti slične znakove; rezultat ostaje kandidat dok se ne potvrdi kroz više kadrova. +- Zvuk u videozapisu uvezenom iz galerije lokalno se dekodira i svodi na RMS dBFS očitanja u prozorima od 250 ms. To omogućuje vremensku korelaciju s vizualnim tragom vozila, ali nije kalibrirano mjerenje razine zvučnog tlaka u dB(A) niti dokaz da je baš opaženo vozilo izvor zvuka. +- Bez kalibracije scene aplikacija ne mjeri stvarnu brzinu vozila. Detekcija, OCR i korelacija buke eksperimentalne su procjene. Ne smiju se predstavljati kao identifikacija osobe, certificirano mjerenje, forenzički dokaz ili automatski zaključak o prometnom prekršaju. + +## Promjena modela + +Pull request koji mijenja model mora navesti izvorni URL, verziju, licencu, SHA-256, skup podataka, ulazno-izlazni ugovor, mapu klasa, pragove i rezultate provjere na stvarnom Android uređaju. Nova binarna datoteka ne prihvaća se bez provjerljivog podrijetla i prava redistribucije. diff --git a/package-lock.json b/package-lock.json index 347eec8..4ccf31e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,13 @@ "jpeg-js": "^0.4.4", "react": "19.2.3", "react-native": "0.86.3", - "react-native-safe-area-context": "~5.7.0" + "react-native-audio-api": "0.13.3", + "react-native-fast-tflite": "^3.0.1", + "react-native-gesture-handler": "~2.32.0", + "react-native-nitro-modules": "^0.37.1", + "react-native-reanimated": "4.5.1", + "react-native-safe-area-context": "~5.7.0", + "react-native-worklets": "0.10.1" }, "devDependencies": { "@types/react": "~19.2.4", @@ -558,6 +564,21 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.29.7.tgz", + "integrity": "sha512-N7zArUXWzAMzm+/N0uPBeVB3Fam5lMxtUwMmDK5f/IBBS7a7p1qeUoxd/6CckXoxUdgsntq1Dh8xNW06maZbDQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-transform-async-generator-functions": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.29.7.tgz", @@ -931,6 +952,38 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-transform-react-pure-annotations": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.29.7.tgz", @@ -947,6 +1000,22 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.29.8.tgz", + "integrity": "sha512-0UpIXPtdDtMXfnV2OJAVMLpj3H/92vmkA6lpSRakmycJvj3VUy6Xs1dM8tXRugupykr5WB+LpiVl0J8LMVg2mg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-transform-runtime": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.29.7.tgz", @@ -976,6 +1045,36 @@ "semver": "bin/semver.js" } }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.29.7.tgz", + "integrity": "sha512-I+WYbGBAiCn7nA6xBrlgPH+MB7HWb4u8pv5S0Pv7OtwNvIFvCCb24YlttKEeUFVurfBCEaOTnuhlqsb7f0Z5Dg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.29.7.tgz", + "integrity": "sha512-NCSEJ4sLFU2gqAub45HYh4fus2yQ36rr6ei6vpU7NdoJqCpxvEG8E6eJpscGyXP3VHD2Ny+fSXr04k1hoUrFqA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-transform-typescript": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.29.7.tgz", @@ -1097,6 +1196,18 @@ "react-native": ">=0.77.3" } }, + "node_modules/@egjs/hammerjs": { + "version": "2.0.17", + "resolved": "https://registry.npmjs.org/@egjs/hammerjs/-/hammerjs-2.0.17.tgz", + "integrity": "sha512-XQsZgjm2EcVUiZQf11UBJQfmZeEmOW8DpI1gsFeln6w0ae0ii4dMQEQ0kjl6DspdWX1aGY1/loyXnP0JS06e/A==", + "license": "MIT", + "dependencies": { + "@types/hammerjs": "^2.0.36" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@emnapi/core": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", @@ -1956,6 +2067,54 @@ "node": "^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0" } }, + "node_modules/@react-native/babel-preset": { + "version": "0.86.3", + "resolved": "https://registry.npmjs.org/@react-native/babel-preset/-/babel-preset-0.86.3.tgz", + "integrity": "sha512-/eqs/Hy9RZRcjdcs4wj3Cqmxvtb3NM5g+Uuh1RIvsjynMO8PRsrVWWLWgBcZL/jYUo+ogXd2NFB0W8L5Bg89xw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/core": "^7.25.2", + "@babel/plugin-proposal-export-default-from": "^7.24.7", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-default-from": "^7.24.7", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-transform-async-generator-functions": "^7.25.4", + "@babel/plugin-transform-async-to-generator": "^7.24.7", + "@babel/plugin-transform-block-scoping": "^7.25.0", + "@babel/plugin-transform-class-properties": "^7.25.4", + "@babel/plugin-transform-classes": "^7.25.4", + "@babel/plugin-transform-destructuring": "^7.24.8", + "@babel/plugin-transform-flow-strip-types": "^7.25.2", + "@babel/plugin-transform-for-of": "^7.24.7", + "@babel/plugin-transform-modules-commonjs": "^7.24.8", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.24.7", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.24.7", + "@babel/plugin-transform-optional-catch-binding": "^7.24.7", + "@babel/plugin-transform-optional-chaining": "^7.24.8", + "@babel/plugin-transform-private-methods": "^7.24.7", + "@babel/plugin-transform-private-property-in-object": "^7.24.7", + "@babel/plugin-transform-react-display-name": "^7.24.7", + "@babel/plugin-transform-react-jsx": "^7.25.2", + "@babel/plugin-transform-react-jsx-self": "^7.24.7", + "@babel/plugin-transform-react-jsx-source": "^7.24.7", + "@babel/plugin-transform-regenerator": "^7.24.7", + "@babel/plugin-transform-runtime": "^7.24.7", + "@babel/plugin-transform-typescript": "^7.25.2", + "@babel/plugin-transform-unicode-regex": "^7.24.7", + "@react-native/babel-plugin-codegen": "0.86.3", + "babel-plugin-syntax-hermes-parser": "0.36.0", + "babel-plugin-transform-flow-enums": "^0.0.2", + "react-refresh": "^0.14.0" + }, + "engines": { + "node": "^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0" + }, + "peerDependencies": { + "@babel/core": "*" + } + }, "node_modules/@react-native/codegen": { "version": "0.86.3", "resolved": "https://registry.npmjs.org/@react-native/codegen/-/codegen-0.86.3.tgz", @@ -2071,6 +2230,41 @@ "node": "^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0" } }, + "node_modules/@react-native/metro-babel-transformer": { + "version": "0.86.3", + "resolved": "https://registry.npmjs.org/@react-native/metro-babel-transformer/-/metro-babel-transformer-0.86.3.tgz", + "integrity": "sha512-0nlwVkG0uT9o72nrGTKUz/IqDnCkM4zE0ihe4r+OJcxH93yBdDm58AXJ3EXZeSjv3osHEFXU7ceajdZiS34mTw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/core": "^7.25.2", + "@react-native/babel-preset": "0.86.3", + "hermes-parser": "0.36.0", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": "^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0" + }, + "peerDependencies": { + "@babel/core": "*" + } + }, + "node_modules/@react-native/metro-config": { + "version": "0.86.3", + "resolved": "https://registry.npmjs.org/@react-native/metro-config/-/metro-config-0.86.3.tgz", + "integrity": "sha512-qdzDMepV2xdUhsO4XC7idnnbt8K6+Hd59AjeK9y4KS86fxSQq5oL0TAbqUuajDGR7beNQoYTuS6AowN/zP0oaQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@react-native/js-polyfills": "0.86.3", + "@react-native/metro-babel-transformer": "0.86.3", + "metro-config": "^0.84.3", + "metro-runtime": "^0.84.3" + }, + "engines": { + "node": "^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0" + } + }, "node_modules/@react-native/normalize-colors": { "version": "0.86.3", "resolved": "https://registry.npmjs.org/@react-native/normalize-colors/-/normalize-colors-0.86.3.tgz", @@ -2475,6 +2669,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/hammerjs": { + "version": "2.0.46", + "resolved": "https://registry.npmjs.org/@types/hammerjs/-/hammerjs-2.0.46.tgz", + "integrity": "sha512-ynRvcq6wvqexJ9brDMS4BnBLzmr0e14d6ZJTEShTBWKymQiHwlAyGu0ZPEFI2Fh1U53F7tN9ufClWM5KvqkKOw==", + "license": "MIT" + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -2526,12 +2726,20 @@ "version": "19.2.18", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", - "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" } }, + "node_modules/@types/react-test-renderer": { + "version": "19.1.0", + "resolved": "https://registry.npmjs.org/@types/react-test-renderer/-/react-test-renderer-19.1.0.tgz", + "integrity": "sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ==", + "license": "MIT", + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/yargs": { "version": "17.0.35", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", @@ -4189,7 +4397,6 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "devOptional": true, "license": "MIT" }, "node_modules/data-view-buffer": { @@ -6612,6 +6819,21 @@ "hermes-estree": "0.36.0" } }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "license": "BSD-3-Clause", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hoist-non-react-statics/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, "node_modules/hosted-git-info": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", @@ -9210,6 +9432,93 @@ } } }, + "node_modules/react-native-audio-api": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/react-native-audio-api/-/react-native-audio-api-0.13.3.tgz", + "integrity": "sha512-U2+vMo7N7CtwHTNNEWnnIckw1EhwhUEmXhQFCL4boo3Q4UfeI1mu2z3DR6BMRL5wWrGYVLNZXQyt5U85nAXntg==", + "license": "MIT", + "dependencies": { + "semver": "^7.7.3" + }, + "bin": { + "setup-rn-audio-api-web": "scripts/setup-rn-audio-api-web.js" + }, + "peerDependencies": { + "react": "*", + "react-native": "*", + "react-native-worklets": ">= 0.6.0" + }, + "peerDependenciesMeta": { + "react-native-worklets": { + "optional": true + } + } + }, + "node_modules/react-native-fast-tflite": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/react-native-fast-tflite/-/react-native-fast-tflite-3.0.1.tgz", + "integrity": "sha512-88wNR/4iR8X0zuQtrpb1jRbF+X+hUqrD8cER4DhNJnbhA+3PuGz8SoP3n8WEhjYWDkGqTme2Ezk+mbeLiiE+6w==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "react": "*", + "react-native": "*", + "react-native-nitro-modules": "*" + } + }, + "node_modules/react-native-gesture-handler": { + "version": "2.32.0", + "resolved": "https://registry.npmjs.org/react-native-gesture-handler/-/react-native-gesture-handler-2.32.0.tgz", + "integrity": "sha512-uYIMOKlKENORq2SABE+jIjbPU+h5I/sQKcq2v16zRq848nwEp1fWRVwML4QWqijc8UcXJC25o54S8GQd4Mf2OA==", + "license": "MIT", + "dependencies": { + "@egjs/hammerjs": "^2.0.17", + "@types/react-test-renderer": "^19.1.0", + "hoist-non-react-statics": "^3.3.0", + "invariant": "^2.2.4" + }, + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, + "node_modules/react-native-is-edge-to-edge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/react-native-is-edge-to-edge/-/react-native-is-edge-to-edge-1.3.1.tgz", + "integrity": "sha512-NIXU/iT5+ORyCc7p0z2nnlkouYKX425vuU1OEm6bMMtWWR9yvb+Xg5AZmImTKoF9abxCPqrKC3rOZsKzUYgYZA==", + "license": "MIT", + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, + "node_modules/react-native-nitro-modules": { + "version": "0.37.1", + "resolved": "https://registry.npmjs.org/react-native-nitro-modules/-/react-native-nitro-modules-0.37.1.tgz", + "integrity": "sha512-KpW6EQVQ/bfegpCGxN9Be+ndvsfj56t4IESEvCASu9gWpJa/NDfHpIeIwCOvZQ3eXmLEj6YD4wCLqcC1FJPr2w==", + "license": "MIT", + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, + "node_modules/react-native-reanimated": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-4.5.1.tgz", + "integrity": "sha512-RnMvtDuR+68ig864gAvZCOdZehqhC5rFmMo0kn+ARfgVSTvFeF6IFLBVgMPUu0KwihaapEyW24WRi6nEyy1kSA==", + "license": "MIT", + "dependencies": { + "react-native-is-edge-to-edge": "^1.3.1", + "semver": "^7.7.3" + }, + "peerDependencies": { + "react": "*", + "react-native": "0.83 - 0.86", + "react-native-worklets": "0.10.x" + } + }, "node_modules/react-native-safe-area-context": { "version": "5.7.0", "resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-5.7.0.tgz", @@ -9220,6 +9529,32 @@ "react-native": "*" } }, + "node_modules/react-native-worklets": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/react-native-worklets/-/react-native-worklets-0.10.1.tgz", + "integrity": "sha512-62mRM19bDpfpdI8HLkEErcdOsrAPDtE9lA/sw+5lLRpzBHNhxaoj9QyY2KjXqUmirelxkX4zuPGTC3VdA0feJA==", + "license": "MIT", + "dependencies": { + "@babel/plugin-transform-arrow-functions": "^7.27.1", + "@babel/plugin-transform-class-properties": "^7.28.6", + "@babel/plugin-transform-classes": "^7.28.6", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.28.6", + "@babel/plugin-transform-optional-chaining": "^7.28.6", + "@babel/plugin-transform-shorthand-properties": "^7.27.1", + "@babel/plugin-transform-template-literals": "^7.27.1", + "@babel/plugin-transform-unicode-regex": "^7.27.1", + "@babel/preset-typescript": "^7.28.5", + "@babel/types": "^7.27.1", + "convert-source-map": "^2.0.0", + "semver": "^7.7.4" + }, + "peerDependencies": { + "@babel/core": "*", + "@react-native/metro-config": "*", + "react": "*", + "react-native": "0.83 - 0.86" + } + }, "node_modules/react-native/node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", diff --git a/package.json b/package.json index 78a9bdc..270c5fc 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,13 @@ "jpeg-js": "^0.4.4", "react": "19.2.3", "react-native": "0.86.3", - "react-native-safe-area-context": "~5.7.0" + "react-native-audio-api": "0.13.3", + "react-native-fast-tflite": "^3.0.1", + "react-native-gesture-handler": "~2.32.0", + "react-native-nitro-modules": "^0.37.1", + "react-native-reanimated": "4.5.1", + "react-native-safe-area-context": "~5.7.0", + "react-native-worklets": "0.10.1" }, "devDependencies": { "@types/react": "~19.2.4", @@ -50,7 +56,8 @@ "doctor": { "reactNativeDirectoryCheck": { "exclude": [ - "@dariyd/react-native-text-recognition" + "@dariyd/react-native-text-recognition", + "react-native-fast-tflite" ] } } diff --git a/src/analysis/audioDecoder.ts b/src/analysis/audioDecoder.ts new file mode 100644 index 0000000..22f0d9b --- /dev/null +++ b/src/analysis/audioDecoder.ts @@ -0,0 +1,19 @@ +import { AUDIO_ANALYSIS_SAMPLE_RATE, pcmChannelsToNoiseSamples } from './audioMetering'; +import type { NoiseSample } from './types'; + +export type DecodedAudioReadings = { + samples: NoiseSample[]; + sampleRateHz: number; + channelCount: number; + durationSeconds: number; +}; + +/** Decodes a local video/audio file on-device and reduces its PCM to 250 ms readings. */ +export async function decodeAudioReadings(mediaUri: string): Promise { + const { decodeAudioData } = await import('react-native-audio-api'); + const buffer = await decodeAudioData(mediaUri, AUDIO_ANALYSIS_SAMPLE_RATE); + const channels = Array.from({ length: buffer.numberOfChannels }, (_, channel) => buffer.getChannelData(channel)); + const samples = pcmChannelsToNoiseSamples(channels, buffer.sampleRate); + if (!samples.length) throw new Error('Audio zapis nema čitljive PCM uzorke.'); + return { samples, sampleRateHz: buffer.sampleRate, channelCount: buffer.numberOfChannels, durationSeconds: buffer.duration }; +} diff --git a/src/analysis/audioMetering.ts b/src/analysis/audioMetering.ts new file mode 100644 index 0000000..b233279 --- /dev/null +++ b/src/analysis/audioMetering.ts @@ -0,0 +1,62 @@ +import type { AudioAnalysisSummary, NoiseSample, NoiseSource } from './types'; + +export const AUDIO_ANALYSIS_SAMPLE_RATE = 8_000; +export const AUDIO_ANALYSIS_WINDOW_MS = 250; +const SILENCE_FLOOR_DBFS = -120; + +function powerToDbfs(power: number) { + if (!Number.isFinite(power) || power <= 0) return SILENCE_FLOOR_DBFS; + return Math.max(SILENCE_FLOOR_DBFS, Math.min(0, 10 * Math.log10(power))); +} + +/** Converts decoded floating-point PCM into compact, time-aligned RMS readings. */ +export function pcmChannelsToNoiseSamples( + channels: Float32Array[], + sampleRate: number, + windowMs = AUDIO_ANALYSIS_WINDOW_MS, +): NoiseSample[] { + if (!channels.length || !Number.isFinite(sampleRate) || sampleRate <= 0) return []; + const length = Math.min(...channels.map(channel => channel.length)); + const windowSize = Math.max(1, Math.round(sampleRate * windowMs / 1_000)); + const samples: NoiseSample[] = []; + + for (let start = 0; start < length; start += windowSize) { + const end = Math.min(length, start + windowSize); + let sumSquares = 0; + let valueCount = 0; + for (const channel of channels) { + for (let index = start; index < end; index += 1) { + const value = channel[index]; + if (!Number.isFinite(value)) continue; + const clamped = Math.max(-1, Math.min(1, value)); + sumSquares += clamped * clamped; + valueCount += 1; + } + } + if (!valueCount) continue; + samples.push({ + timeMs: Math.round(((start + end) / 2 / sampleRate) * 1_000), + dbfs: powerToDbfs(sumSquares / valueCount), + }); + } + return samples; +} + +export function summarizeNoiseSamples( + samples: NoiseSample[], + source: NoiseSource, + sampleRateHz?: number, +): AudioAnalysisSummary | undefined { + const valid = samples.filter(sample => Number.isFinite(sample.timeMs) && Number.isFinite(sample.dbfs)); + if (!valid.length) return undefined; + const averagePower = valid.reduce((sum, sample) => sum + 10 ** (sample.dbfs / 10), 0) / valid.length; + return { + source, + sampleCount: valid.length, + sampleRateHz, + windowMs: AUDIO_ANALYSIS_WINDOW_MS, + averageDbfs: powerToDbfs(averagePower), + peakDbfs: Math.max(...valid.map(sample => Math.min(0, sample.dbfs))), + note: 'RMS očitanje iz audio zapisa; dBFS nije kalibrirana razina zvučnog tlaka u dB(A).', + }; +} diff --git a/src/analysis/sessionAnalysis.ts b/src/analysis/sessionAnalysis.ts index fa85d9f..2ca572b 100644 --- a/src/analysis/sessionAnalysis.ts +++ b/src/analysis/sessionAnalysis.ts @@ -1,10 +1,11 @@ import { extractEvidenceFrames, extractEvidenceFramesAtTimes } from './evidenceFrames'; import * as FileSystem from 'expo-file-system/legacy'; -import { rankVehicleEvidenceFrames } from './frameRanking'; +import { rankEvidenceFrames, rankVehicleEvidenceFrames } from './frameRanking'; import { trackVehicles } from './tracking'; import { detectVehiclesInFrames, vehicleDetectionAvailable } from './vehicleDetectionModel'; -import type { AnalysisReport, NoiseSample } from './types'; +import type { AnalysisReport, NoiseSample, NoiseSource } from './types'; import { prepareVehicleAnalysis } from './vehicleAnalysis'; +import { decodeAudioReadings } from './audioDecoder'; export type CaptureLocation = { latitude: number; @@ -21,7 +22,7 @@ export type SessionAnalysis = { report?: AnalysisReport; }; -export type AnalysisProgress = 'Izdvajam kadrove' | 'Tražim vozila' | 'Izdvajam guste kadrove' | 'Rangiram dokaze' | 'Čitam oznake'; +export type AnalysisProgress = 'Dekodiram zvuk' | 'Izdvajam kadrove' | 'Tražim vozila' | 'Izdvajam guste kadrove' | 'Rangiram dokaze' | 'Čitam oznake'; const DENSE_SAMPLE_INTERVAL_MS = 200; const VEHICLE_WINDOW_PADDING_MS = 1_000; @@ -56,8 +57,26 @@ function denseVehicleTimes(detections: Awaited void): Promise { + let effectiveNoiseSamples = noiseSamples; + let noiseSource: NoiseSource = 'live-metering'; + let audioSampleRateHz: number | undefined; + let audioWarning: string | undefined; + if (!effectiveNoiseSamples.length) { + onProgress?.('Dekodiram zvuk'); + try { + const decoded = await decodeAudioReadings(sessionUri); + effectiveNoiseSamples = decoded.samples; + noiseSource = 'embedded-video'; + audioSampleRateHz = decoded.sampleRateHz; + console.log('[JEKA AOPS] Audio je dekodiran', { sessionId, samples: decoded.samples.length, channels: decoded.channelCount, sampleRateHz: decoded.sampleRateHz, durationSeconds: decoded.durationSeconds }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + audioWarning = 'Audio zapis nije dostupan ili ga ovaj uređaj ne može dekodirati.'; + console.warn('[JEKA AOPS] Audio nije dekodiran', { sessionId, error: message }); + } + } if (!vehicleDetectionAvailable) { - const result = await prepareVehicleAnalysis(sessionUri, [], noiseSamples); + const result = await prepareVehicleAnalysis(sessionUri, [], effectiveNoiseSamples, false, undefined, undefined, noiseSource, audioSampleRateHz, audioWarning); return { status: 'ready-for-model', updatedAt: new Date().toISOString(), @@ -87,10 +106,18 @@ export async function beginAutomaticAnalysis(sessionUri: string, sessionId: stri const denseDetections = await detectVehiclesInFrames(denseCandidates); const denseFrameTimesWithVehicles = new Set(denseDetections.map(detection => detection.frameTimeMs)); const denseVehicleCandidates = denseCandidates.filter(frame => denseFrameTimesWithVehicles.has(frame.frameTimeMs)); - const evidencePool = denseVehicleCandidates.length ? denseVehicleCandidates : vehicleCandidates; + // If the detector misses every vehicle, retain the best full frames and run + // OCR as an explicitly unassigned fallback instead of returning no evidence. + const evidencePool = denseVehicleCandidates.length + ? denseVehicleCandidates + : vehicleCandidates.length + ? vehicleCandidates + : candidates; const detectionPool = denseVehicleCandidates.length ? denseDetections : candidateDetections; onProgress?.('Rangiram dokaze'); - const rankedFrames = await rankVehicleEvidenceFrames(evidencePool, detectionPool); + const rankedFrames = detectionPool.length + ? await rankVehicleEvidenceFrames(evidencePool, detectionPool) + : await rankEvidenceFrames(evidencePool); const tracks = trackVehicles(detectionPool, evidencePool); const selectedIds = new Set(); for (const track of tracks) { @@ -133,10 +160,13 @@ export async function beginAutomaticAnalysis(sessionUri: string, sessionId: stri const result = await prepareVehicleAnalysis( sessionUri, evidenceFrames, - noiseSamples, + effectiveNoiseSamples, vehicleCandidates.length === 0, selectedDetections, tracks, + noiseSource, + audioSampleRateHz, + audioWarning, ); console.log('[JEKA AOPS] Izvještaj analize', { sessionId, @@ -149,7 +179,7 @@ export async function beginAutomaticAnalysis(sessionUri: string, sessionId: stri status: result.status === 'completed' ? 'completed' : result.status === 'ready-for-model' ? 'ready-for-model' : 'failed', updatedAt: new Date().toISOString(), note: vehicleCandidates.length === 0 - ? 'U izdvojenim kadrovima nisu pronađena vozila; prazni kadrovi nisu analizirani.' + ? `Detektor nije pronašao vozilo; OCR je ipak provjeren na ${evidenceFrames.length} najbolja kadra.` : result.status === 'completed' ? `Analiza je dovršena na ${evidenceFrames.length} rangiranih dokaznih kadrova.` : result.status === 'ready-for-model' diff --git a/src/analysis/tfliteImage.ts b/src/analysis/tfliteImage.ts new file mode 100644 index 0000000..7c9161e --- /dev/null +++ b/src/analysis/tfliteImage.ts @@ -0,0 +1,90 @@ +import type { BoundingBox, VehicleDetection } from './types'; + +export const DETECTOR_INPUT_SIZE = 320; +export const VEHICLE_CLASS_IDS = new Set([1, 2, 3, 5, 7]); + +export type LetterboxTransform = { + inputSize: number; + scaledWidth: number; + scaledHeight: number; + padX: number; + padY: number; +}; + +const clamp = (value: number) => Math.max(0, Math.min(1, value)); + +/** Converts decoded RGBA pixels to a square RGB tensor without stretching the scene. */ +export function letterboxRgbaToRgb( + rgba: Uint8Array, + width: number, + height: number, + inputSize = DETECTOR_INPUT_SIZE, +) { + if (width <= 0 || height <= 0 || rgba.length < width * height * 4) { + throw new Error('Kadar nema valjane dimenzije za detekciju.'); + } + const scale = Math.min(inputSize / width, inputSize / height); + const scaledWidth = Math.max(1, Math.round(width * scale)); + const scaledHeight = Math.max(1, Math.round(height * scale)); + const padX = Math.floor((inputSize - scaledWidth) / 2); + const padY = Math.floor((inputSize - scaledHeight) / 2); + const rgb = new Uint8Array(inputSize * inputSize * 3); + rgb.fill(114); + + for (let targetY = 0; targetY < scaledHeight; targetY += 1) { + const sourceY = Math.min(height - 1, Math.floor((targetY + 0.5) / scale)); + for (let targetX = 0; targetX < scaledWidth; targetX += 1) { + const sourceX = Math.min(width - 1, Math.floor((targetX + 0.5) / scale)); + const sourceOffset = (sourceY * width + sourceX) * 4; + const targetOffset = ((targetY + padY) * inputSize + targetX + padX) * 3; + rgb[targetOffset] = rgba[sourceOffset]; + rgb[targetOffset + 1] = rgba[sourceOffset + 1]; + rgb[targetOffset + 2] = rgba[sourceOffset + 2]; + } + } + return { rgb, transform: { inputSize, scaledWidth, scaledHeight, padX, padY } satisfies LetterboxTransform }; +} + +/** Maps a model [top,left,bottom,right] box back to normalized source-frame coordinates. */ +export function modelBoxToSourceBox(box: ArrayLike, transform: LetterboxTransform): BoundingBox | undefined { + if (box.length < 4) return undefined; + const [top, left, bottom, right] = [Number(box[0]), Number(box[1]), Number(box[2]), Number(box[3])]; + if (![top, left, bottom, right].every(Number.isFinite)) return undefined; + const x1 = clamp((left * transform.inputSize - transform.padX) / transform.scaledWidth); + const y1 = clamp((top * transform.inputSize - transform.padY) / transform.scaledHeight); + const x2 = clamp((right * transform.inputSize - transform.padX) / transform.scaledWidth); + const y2 = clamp((bottom * transform.inputSize - transform.padY) / transform.scaledHeight); + const width = x2 - x1; + const height = y2 - y1; + if (width < 0.015 || height < 0.015 || width * height < 0.0008) return undefined; + return { x: x1, y: y1, width, height }; +} + +export function parseEfficientDetOutputs( + outputs: ArrayBuffer[], + transform: LetterboxTransform, + frameTimeMs: number, + threshold = 0.32, +): VehicleDetection[] { + if (outputs.length < 4) throw new Error(`Model je vratio ${outputs.length} izlaza umjesto očekivana 4.`); + const boxes = new Float32Array(outputs[0]); + const classes = new Float32Array(outputs[1]); + const scores = new Float32Array(outputs[2]); + const detectedCount = new Float32Array(outputs[3]); + const count = Math.min( + Math.max(0, Math.round(detectedCount[0] ?? 0)), + classes.length, + scores.length, + Math.floor(boxes.length / 4), + ); + const detections: VehicleDetection[] = []; + for (let index = 0; index < count; index += 1) { + const confidence = scores[index]; + const classId = Math.round(classes[index]); + if (!Number.isFinite(confidence) || confidence < threshold || !VEHICLE_CLASS_IDS.has(classId)) continue; + const boundingBox = modelBoxToSourceBox(boxes.subarray(index * 4, index * 4 + 4), transform); + if (!boundingBox) continue; + detections.push({ label: 'vehicle', confidence, frameTimeMs, boundingBox }); + } + return detections; +} diff --git a/src/analysis/types.ts b/src/analysis/types.ts index 9f024d4..8b16592 100644 --- a/src/analysis/types.ts +++ b/src/analysis/types.ts @@ -60,6 +60,18 @@ export type NoiseSample = { dbfs: number; }; +export type NoiseSource = 'embedded-video' | 'live-metering'; + +export type AudioAnalysisSummary = { + source: NoiseSource; + sampleCount: number; + sampleRateHz?: number; + windowMs: number; + averageDbfs: number; + peakDbfs: number; + note: string; +}; + export type VehicleTrack = { id: string; detections: VehicleDetection[]; @@ -79,6 +91,7 @@ export type AnalysisReport = { evidenceFrames: EvidenceFrame[]; vehicleTracks: VehicleTrack[]; unassignedPlateCandidates: PlateCandidate[]; + audioSummary?: AudioAnalysisSummary; limitations: string[]; }; diff --git a/src/analysis/vehicleAnalysis.ts b/src/analysis/vehicleAnalysis.ts index 1892fd1..ecccb12 100644 --- a/src/analysis/vehicleAnalysis.ts +++ b/src/analysis/vehicleAnalysis.ts @@ -1,12 +1,13 @@ -import type { AnalysisReport, EvidenceFrame, NoiseSample, VehicleAnalysisResult, VehicleTrack } from './types'; +import type { AnalysisReport, EvidenceFrame, NoiseSample, NoiseSource, VehicleAnalysisResult, VehicleTrack } from './types'; import { trackVehicles } from './tracking'; import { detectVehiclesInFrames } from './vehicleDetectionModel'; import { recognizePlateObservations } from './plateOcr'; import { associatePlatesToTracks } from './plateAssociation'; import { correlateNoiseToTrack } from './noiseCorrelation'; import { attachVehicleEvidenceCrops } from './vehicleEvidence'; +import { summarizeNoiseSamples } from './audioMetering'; -function createEmptyReport(sessionUri: string, evidenceFrames: EvidenceFrame[], tracks: AnalysisReport['vehicleTracks'] = [], plateCandidates: AnalysisReport['unassignedPlateCandidates'] = [], modelError?: string, ocrError?: string): AnalysisReport { +function createEmptyReport(sessionUri: string, evidenceFrames: EvidenceFrame[], tracks: AnalysisReport['vehicleTracks'] = [], plateCandidates: AnalysisReport['unassignedPlateCandidates'] = [], modelError?: string, ocrError?: string, noiseSamples: NoiseSample[] = [], noiseSource: NoiseSource = 'live-metering', audioSampleRateHz?: number, audioWarning?: string): AnalysisReport { return { version: 1, sessionUri, @@ -15,6 +16,7 @@ function createEmptyReport(sessionUri: string, evidenceFrames: EvidenceFrame[], evidenceFrames, vehicleTracks: tracks, unassignedPlateCandidates: plateCandidates, + audioSummary: summarizeNoiseSamples(noiseSamples, noiseSource, audioSampleRateHz), limitations: [ ...(modelError ? [`Detekcija vozila nije pokrenuta: ${modelError}`] : ['Detekcija vozila koristi početni COCO model; rezultat nije konačna identifikacija vozila.']), ...(ocrError ? [`OCR nije pokrenut: ${ocrError}`] : ['OCR rezultat je samo kandidat dok se ne potvrdi kroz više kadrova i prostorno ne veže uz vozilo.']), @@ -22,6 +24,7 @@ function createEmptyReport(sessionUri: string, evidenceFrames: EvidenceFrame[], ...(plateCandidates.length ? ['OCR kandidati nisu prostorno pridruženi pojedinom vozilu; ne predstavljaju potvrđenu registracijsku oznaku.'] : []), 'Bez potvrde kroz više kadrova ne prikazuje se očitana registracijska oznaka.', 'Mjerenje zvuka u dBFS nije kalibrirano mjerenje zvučnog tlaka u dB(A).', + ...(audioWarning ? [audioWarning] : []), ], }; } @@ -34,6 +37,9 @@ export async function prepareVehicleAnalysis( noVehicleFound = false, precomputedDetections?: VehicleAnalysisResult['detections'], precomputedTracks?: VehicleTrack[], + noiseSource: NoiseSource = 'live-metering', + audioSampleRateHz?: number, + audioWarning?: string, ): Promise { let detections: VehicleAnalysisResult['detections'] = []; let plateCandidates: AnalysisReport['unassignedPlateCandidates'] = []; @@ -69,9 +75,16 @@ export async function prepareVehicleAnalysis( modelError = noVehicleFound ? undefined : 'Nema dostupnih dokaznih kadrova.'; } - const report = createEmptyReport(sessionUri, evidenceFrames, tracks, plateCandidates, modelError, ocrError); + const report = createEmptyReport(sessionUri, evidenceFrames, tracks, plateCandidates, modelError, ocrError, noiseSamples, noiseSource, audioSampleRateHz, audioWarning); if (noVehicleFound) { - report.limitations = ['U snimci nisu pronađena vozila; kadrovi bez vozila nisu uključeni u analizu.']; + report.limitations = [ + 'Model u odabranim kadrovima nije pronašao vozilo.', + 'OCR je svejedno pokrenut na najboljim cijelim kadrovima; kandidati bez okvira vozila ostaju nepridruženi.', + ...report.limitations.filter(item => !item.startsWith('Detekcija vozila koristi')), + ]; + } + if (!noiseSamples.length) { + report.limitations.push('Nema audio uzoraka za očitanje i vremensku korelaciju s vozilom.'); } return { status: modelError ? 'ready-for-model' : 'completed', diff --git a/src/analysis/vehicleDetectionModel.ts b/src/analysis/vehicleDetectionModel.ts index fea3f55..a02c405 100644 --- a/src/analysis/vehicleDetectionModel.ts +++ b/src/analysis/vehicleDetectionModel.ts @@ -1,19 +1,39 @@ +import { toByteArray } from 'base64-js'; +import * as FileSystem from 'expo-file-system/legacy'; +import { decode } from 'jpeg-js'; +import type { TfliteModel } from 'react-native-fast-tflite'; import type { EvidenceFrame, VehicleDetection } from './types'; +import { letterboxRgbaToRgb, parseEfficientDetOutputs } from './tfliteImage'; -export const vehicleDetectionAvailable = false; +export const vehicleDetectionAvailable = true; +const MODEL_ASSET = require('../../assets/models/efficientdet-lite0-int8-v1.tflite'); +let modelPromise: Promise | undefined; -/** - * Public-source fallback. - * - * The previous repository snapshot bundled a TFLite binary whose exact source - * and redistribution terms could not be proven. The binary and its loader are - * intentionally excluded from the public distribution. Contributors can add a - * detector through this stable function boundary after documenting the model's - * source, license, checksum, input contract and output contract in docs/MODEL.md. - */ -export async function detectVehiclesInFrames(_frames: EvidenceFrame[]): Promise { - if (_frames.length) { - throw new Error('Model detekcije vozila nije uključen u ovu javnu verziju aplikacije.'); +async function loadModel() { + modelPromise ??= import('react-native-fast-tflite').then(({ loadTensorflowModel }) => + loadTensorflowModel(MODEL_ASSET, []), + ); + const model = await modelPromise; + const input = model.inputs[0]; + if (!input || input.dataType !== 'uint8' || input.shape.join('x') !== '1x320x320x3') { + throw new Error(`Neočekivani ulaz detektora: ${input?.dataType ?? 'nepoznat'} ${input?.shape.join('x') ?? ''}`.trim()); } - return []; + return model; +} + +/** Runs the fixed, locally bundled EfficientDet-Lite0 model on extracted JPEG evidence frames. */ +export async function detectVehiclesInFrames(frames: EvidenceFrame[]): Promise { + if (!frames.length) return []; + const model = await loadModel(); + const detections: VehicleDetection[] = []; + for (const frame of frames) { + if (!frame.uri) continue; + const base64 = await FileSystem.readAsStringAsync(frame.uri, { encoding: FileSystem.EncodingType.Base64 }); + const image = decode(toByteArray(base64), { useTArray: true }); + const { rgb, transform } = letterboxRgbaToRgb(image.data, image.width, image.height); + const input = rgb.buffer.slice(rgb.byteOffset, rgb.byteOffset + rgb.byteLength) as ArrayBuffer; + const outputs = await model.run([input]); + detections.push(...parseEfficientDetOutputs(outputs, transform, frame.frameTimeMs)); + } + return detections; } diff --git a/src/services/reportService.ts b/src/services/reportService.ts index 79fa2b6..8cf9059 100644 --- a/src/services/reportService.ts +++ b/src/services/reportService.ts @@ -17,15 +17,21 @@ export async function shareRecording(session: Session) { export async function shareSessionReport(session: Session, progress?: string) { await ensureSharingAvailable(); const report = session.analysis?.report; + const audioAverage = report?.audioSummary?.averageDbfs ?? session.noiseAverageDbfs; + const audioPeak = report?.audioSummary?.peakDbfs ?? session.noisePeakDbfs; const lines = [ 'JEKA AOPS — izvještaj prometne sesije', `Vrijeme snimanja: ${new Date(session.createdAt).toLocaleString('hr-HR')}`, `Trajanje: ${formatDuration(session.durationSeconds)}`, `Lokacija: ${formatLocation(session.location)}`, - `Buka: prosjek ${formatDbfs(session.noiseAverageDbfs)}, vrh ${formatDbfs(session.noisePeakDbfs)} (dBFS)`, + `Zvuk: prosjek ${formatDbfs(audioAverage)}, vrh ${formatDbfs(audioPeak)} (dBFS)`, + ...(report?.audioSummary ? [`Dekodirani audio: ${report.audioSummary.sampleCount} očitanja po ${report.audioSummary.windowMs} ms`] : []), `Status obrade: ${analysisLabel(session.analysis, progress)}`, `Dokazni kadrovi: ${report?.evidenceFrames.length ?? 0}`, `Tragovi vozila: ${report?.vehicleTracks.length ?? 0}`, + ...(report?.vehicleTracks.flatMap(track => track.noise ? [ + `Vozilo ${track.id}: zvuk uz prolazak ${formatDbfs(track.noise.averageDbfs)} prosjek, ${formatDbfs(track.noise.peakDbfs)} vrh`, + ] : []) ?? []), '', 'Ograničenja i upozorenja:', ...(report?.limitations ?? ['Automatska analiza još nije pripremljena.']).map(item => `- ${item}`), diff --git a/tests/audioMetering.test.ts b/tests/audioMetering.test.ts new file mode 100644 index 0000000..50a8173 --- /dev/null +++ b/tests/audioMetering.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { pcmChannelsToNoiseSamples, summarizeNoiseSamples } from '../src/analysis/audioMetering'; + +describe('audio metering', () => { + it('converts full-scale PCM to 0 dBFS in time-aligned windows', () => { + const samples = pcmChannelsToNoiseSamples([new Float32Array(8).fill(1)], 8, 500); + expect(samples).toHaveLength(2); + expect(samples[0]).toEqual({ timeMs: 250, dbfs: 0 }); + expect(samples[1].timeMs).toBe(750); + }); + + it('calculates RMS across channels and uses a safe silence floor', () => { + const halfScale = pcmChannelsToNoiseSamples([new Float32Array(4).fill(0.5), new Float32Array(4).fill(-0.5)], 4, 1_000); + expect(halfScale[0].dbfs).toBeCloseTo(-6.0206, 3); + expect(pcmChannelsToNoiseSamples([new Float32Array(4)], 4, 1_000)[0].dbfs).toBe(-120); + }); + + it('summarizes readings by acoustic power rather than averaging decibels', () => { + const summary = summarizeNoiseSamples([{ timeMs: 125, dbfs: 0 }, { timeMs: 375, dbfs: -120 }], 'embedded-video', 8_000); + expect(summary?.averageDbfs).toBeCloseTo(-3.0103, 3); + expect(summary?.peakDbfs).toBe(0); + expect(summary?.sampleCount).toBe(2); + }); +}); diff --git a/tests/tfliteImage.test.ts b/tests/tfliteImage.test.ts new file mode 100644 index 0000000..5374654 --- /dev/null +++ b/tests/tfliteImage.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; +import { letterboxRgbaToRgb, modelBoxToSourceBox, parseEfficientDetOutputs } from '../src/analysis/tfliteImage'; + +const buffer = (values: number[]) => new Float32Array(values).buffer; + +describe('TFLite predobrada kadra', () => { + it('zadržava omjer slike i dodaje rubove bez rastezanja', () => { + const rgba = new Uint8Array(4 * 2 * 4).fill(255); + const { rgb, transform } = letterboxRgbaToRgb(rgba, 4, 2, 8); + expect(rgb).toHaveLength(8 * 8 * 3); + expect(transform).toEqual({ inputSize: 8, scaledWidth: 8, scaledHeight: 4, padX: 0, padY: 2 }); + expect([...rgb.slice(0, 3)]).toEqual([114, 114, 114]); + expect([...rgb.slice((2 * 8) * 3, (2 * 8) * 3 + 3)]).toEqual([255, 255, 255]); + }); + + it('vraća okvir iz letterbox koordinata u izvorni kadar', () => { + const box = modelBoxToSourceBox([0.25, 0.25, 0.75, 0.75], { + inputSize: 320, + scaledWidth: 320, + scaledHeight: 180, + padX: 0, + padY: 70, + }); + expect(box?.x).toBeCloseTo(0.25); + expect(box?.width).toBeCloseTo(0.5); + expect(box?.y).toBeCloseTo(1 / 18); + expect(box?.height).toBeCloseTo(8 / 9); + }); +}); + +describe('EfficientDet izlazi', () => { + it('zadržava cestovna vozila i odbacuje druge COCO klase', () => { + const outputs = [ + buffer([0.1, 0.2, 0.8, 0.7, 0.2, 0.2, 0.6, 0.6]), + buffer([2, 0]), // car, person + buffer([0.91, 0.99]), + buffer([2]), + ]; + const detections = parseEfficientDetOutputs(outputs, { + inputSize: 320, + scaledWidth: 320, + scaledHeight: 320, + padX: 0, + padY: 0, + }, 1_500); + expect(detections).toHaveLength(1); + expect(detections[0]).toMatchObject({ label: 'vehicle', frameTimeMs: 1_500, confidence: expect.closeTo(0.91) }); + expect(detections[0].boundingBox).toMatchObject({ x: expect.closeTo(0.2), y: expect.closeTo(0.1) }); + }); + + it('odbacuje slabe detekcije', () => { + const detections = parseEfficientDetOutputs([ + buffer([0.1, 0.1, 0.8, 0.8]), buffer([2]), buffer([0.2]), buffer([1]), + ], { inputSize: 320, scaledWidth: 320, scaledHeight: 320, padX: 0, padY: 0 }, 0); + expect(detections).toEqual([]); + }); +});