Merge pull request #193 from ptidejteam/develop

Pre-release checks

Author: nkinesis

.github/workflows/maven.yml

@@ -10,12 +10,12 @@ name: Ptidej Middleware CI with Maven
 
 on:
   push:
-    branches: [ "master" ]
+    branches: [ "master", "develop" ]
     paths-ignore:
       - 'docs/**'
       - '*.md'
   pull_request:
-    branches: [ "master" ]
+    branches: [ "master", "develop" ]
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -37,9 +37,9 @@ jobs:
       - name: Generate keypair for authentication (in correct location)
         working-directory: Middleware
         run: |
-          mkdir -p src/main/resources/certs
-          openssl genpkey -algorithm RSA -out src/main/resources/certs/private.pem -pkeyopt rsa_keygen_bits:2048
-          openssl rsa -pubout -in src/main/resources/certs/private.pem -out src/main/resources/certs/public.pem
+          mkdir -p src/main/resources/scripts/certs/
+          openssl genpkey -algorithm RSA -out src/main/resources/scripts/certs/private.pem -pkeyopt rsa_keygen_bits:2048
+          openssl rsa -pubout -in src/main/resources/scripts/certs/private.pem -out src/main/resources/scripts/certs/public.pem
 
       - name: Clean and test with Maven
         working-directory: Middleware
@@ -52,4 +52,3 @@ jobs:
       - name: Build with Maven
         run: mvn install
         working-directory: Middleware
-

.gitignore

@@ -235,4 +235,6 @@ cython_debug/
 #########
 .DS_Store
 
-certs/
+certs/
+
+credentials/

Middleware/.classpath

@@ -48,5 +48,16 @@
 			<attribute name="test" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry kind="src" path=".apt_generated">
+		<attributes>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path=".apt_generated_tests">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

Middleware/.gitignore

@@ -51,4 +51,13 @@ build/
 !.mvn/wrapper/maven-wrapper.jar
 !**/src/main/**/target/
 !**/src/test/**
+!META-INF/**
+
+## Ignore JWT certs/keys
+Middleware/src/main/resources/scripts/certs/
+
+## Ignore user credentials
+Middleware/src/main/resources/scripts/credentials/
+Middleware/src/main/resources/credentials/
+
 

Middleware/.project

@@ -5,6 +5,11 @@
 	<projects>
 	</projects>
 	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
 		<buildCommand>
 			<name>org.eclipse.jdt.core.javabuilder</name>
 			<arguments>
@@ -19,6 +24,7 @@
 	<natures>
 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
 		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.python.pydev.pythonNature</nature>
 	</natures>
 	<filteredResources>
 		<filter>

Middleware/DATA_SOURCES.md

@@ -0,0 +1,54 @@
+# Dataset Catalog
+
+## Overview
+This document provides a comprehensive catalog of datasets available for research and analysis, covering building information, transportation, energy consumption, and urban infrastructure in Montreal and surrounding areas. Currently, these datasets are not in the repository, they will be added in coming days. 
+
+---
+
+## Datasets
+
+
+| ID | Dataset Names | Description | Format |
+|---|---|---|---|
+| DS1 | Montreal Districts/Boroughs | Polygons of Montreal districts and boroughs sourced from City of Montreal. | GeoJSON |
+| DS2 | Montreal Building Geometry (LoD2) | 3D building footprints extruded by average height, represented as GeoJSON feature‑collection. | RAR (GeoJSON) |
+| DS3 | Montreal Buildings (LoD1) | 2D polygons and metadata of Montreal buildings derived from property assessment rolls. | SHP |
+| DS4 | Canadian Building Footprints | ~2.5M building footprints for Quebec, generated and licensed by Microsoft. | ZIP / GeoJSON |
+| DS5 | MTL Trajet_routes | Mobile-phone trip trajectories in Montreal collected in 2016–17. | ZIP |
+| DS6 | Transportation in Montreal Tweets | Tweets about transportation in Montreal, with multiple tweet attributes. | CSV |
+| DS7 | Census tract boundaries | Geometric boundaries of Canadian census tract units. | CSV / ZIP / PDF |
+| DS8 | Varennes Digital Surface Model (DSM) | DSM .tif of Varennes area used for data cleaning around Varennes Public Library. | TIFF |
+| DS9 | Varennes Digital Terrain Model (DTM) | DTM .tif used in the same context. | TIFF |
+| DS10 | Varennes Boundary Shape File | Shapefile boundary covering ~5,229 buildings around Varennes Public Library. | SHP |
+| DS11 | Geoindex shape file clipped for Varennes | Shapefile clipped based on a defined Varennes boundary using PyQGIS. | SHP |
+| DS12 | Whole building LCA | Harmonized North America-wide whole-building life-cycle assessment dataset (first release). | ZIP / XLSX |
+| DS13 | Concordia Building Blueprints | Blueprints of all Concordia campus buildings (SGW + Loyola) Campus | DWG, PDF, PNG/JPG |
+| DS14 | Hydro Quebec Energy Metering Data | Comprehensive dataset containing energy metering data of Hydro Quebec users | Parquet |
+| DS15 | Building Elecricity Data | Comes from Concordia Facilities Management - Weekly basis | CSV |
+| DS16 | General Transit Feed Specification | The General Transit Feed Specification (GTFS) is a standardized open data portal that offers information about transit systems. | TXT files |
+| DS17 | Lot boundaries of Montreal Chinatown 2014 | Buildings' lot boundaries and associated metadata for Chinatown, Montreal. This dataset pertains to the year 2014. | JSON |
+| DS18 | Montreal Chinatown buildings footprints | Polygon footprints of buildings in Montreal's Chinatown. | GeoJSON |
+| DS19 | Montreal Chinatown points of interests | Locations and information about Points of Interest in Montreal's Chinatown. | JSON |
+| DS20 | Public trees in Montreal chinatown | Positions and information of trees planted in Montreal's Chinatown. | JSON |
+| DS21 | Public Parking Meters Montreal Chinatown | Geographical information and details about parking meter spaces available in Montreal's Chinatown. | GeoJSON |
+| DS22 | Chinatown Polygon Boundary 2022 | Data on Montreal's Chinatown boundary polygon declared in 2022. | JSON |
+| DS23 | CERC CMM | This is the first version of the CERC's Communauté métropolitaine de Montréal (CMM). The map was developed based on different datasets. | ZIP |
+| DS24 | CERC CMM: One feature for each building | This is the second version of the CERC's Communauté métropolitaine de Montréal (CMM). This data layer (in GeoJSON format) is composed in a way that each feature represents one building. | CKAN | GEOJSON |
+| DS25 | Property assessment units | Property Assessment Units (Unités d'évaluation foncière" ) contains Vector geospatial data of property divisions within the Montreal agglomeration, providing general information on property assessment units. This includes codification (CUBF), approximate dimensions, and registration numbers. | JSON/CSV |
+| DS26 | BIXI Stations | "BIXI Stations" lists stations within the BIXI Montréal self-service bicycle network, providing information on their geographic positions, available bicycles, and terminals. BIXI Montréal has published the dataset. | CKAN | CSV |
+| DS27 | BIXI Trips History (2021, 2022) | BIXI trip history contains detailed information about trips taken using the BIXI Montréal self-service bicycle network in 2021. BIXI is a popular bike-sharing system that offers a convenient and eco-friendly mode of transportation for residents and visitors. | ZIP |
+| DS28 | Load data by a load generator tool | Information missing | XLSX |
+| DS29 | Montreal downtown buildings (SGW campus) | This data includes 188 buildings in Montreal downtown. All of the SGW campus building of Concordia University is included. | GeoJSON |
+| DS30 | Varennes shp files | This .tif file is a Digital Surface Model (DSM) of Varennes. It is being used for a data cleaning and updating project focused on buildings around the Varennes Public Library. Metadata can also be found in the .rar file. High-Resolution Digital Elevation Model (HRDEM) - CanElevation Series | RAR/PDF |
+| DS31 | Varennes-Clipped Quebec Property Assessment Rolls 2022 | This shape file is from Quebec property assessment rolls 2022 data. It has been clipped based on buildings around Varennes Public Library. | RAR |
+| DS32 | Comptages Vehicules Cyclistes Pietons | Count of Motor Vehicles, Cyclists, and Pedestrians at Montreal Intersections Counts of vehicles, cyclists, and pedestrians are available for most intersections equipped with traffic lights and specific intersections where the installation of lights was under study. | CSV |
+| DS33 | Hydro Quebec Dataset for Concordia Buildings | Yearly bills - SGW and Loyola Campus | CSV |
+| DS34 | Monthly billing data | Monthly billing consumption data - several swimming poola arenas and fire station in Montreal | CSV |
+| DS35 | Some dataset | Some csv files - a lot of them | CSV |
+| DS36 | Occupancy Data | Occupancy sensors data - 14th floor - EV building (4 10 9) | API Call/CSV |
+| DS37 | GENOME Project 2 dataset | Data Genome Project 2 - Clean version of electricity consumption - 1630 institutional buildings across north america | CSV |
+| DS38 | Commuters Survey | Commuters survey | Concordia sustainability office | Information missing |
+| DS39 | Synthetic population dataset | The dataset represents a synthetic population developed by Polytechnique Montréal. Based on Home Travel Survey data. Covers 100% of the population of Montreal, capturing detailed travel behavior. | XML |
+| DS40 | PECAN Street Residential Building Dataset in the California Region | Historical time series load data recorded from the real-world PECAN street in the California region. | CSV |
+| DS41 | Data Ontology CIMNE-CERC | Information missing | Information missing |
+| DS42 | Buildings demands energy generation from Solar PV systems | Net electricity load, and net demand (load -PV). The data also includes blueprints of architectural drawings, road maps, and types of electrical demands and their specifications. Data of 5 buildings. | DWG, PDF, PNG/JPG, CSV |

Middleware/README.md

@@ -36,7 +36,7 @@ To collaborate with CITYdata, you can use the Java IDE of your choice. The CITYd
 
 Before generating the public and private keys, ensure you have `openssl` installed in your machine!
 
-- Step 1: Create a folder named `certs` inside the `src/main/resources` directory.
+- Step 1: Create a folder named `certs` inside the `src/main/resources/scripts/certs` directory.
 - Step 2: Navigate to the `certs` folder and run the following commands to generate your public and private keys:
 
 ```bash
@@ -46,6 +46,28 @@ openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in keypair.pem -out priv
 ```
 - Step 3: Once public and private keys are generated, you can delete the keypair.pem
 
+## Adding Users
+
+Before proceeding, ensure the following prerequisites are met:
+* `Python` is installed and added to your system's environment variables.
+* The `bcrypt` package is installed. If not, open your terminal (Bash) and run:
+
+	pip install bcrypt
+
+Once the above requirements are fulfilled, follow these steps to add, remove, or update users:
+
+- Open `Bash` and navigate to the following directory in the `citydata` project:
+
+	cd src/main/resources/scripts
+
+- Run the credentials manager script:
+
+	./credentials-manager.sh
+
+- An interactive menu will appear, allowing you to add, remove, or update users in `citydata`.
+
+- If you wish to use the instance of CITYdata hosted at the NGCI, please contact us for user creation.
+
 ## How do I set it up?
 
 - Install Java and Maven in your operating system
@@ -79,7 +101,7 @@ java -jar ./target/Middleware-0.0.1-SNAPSHOT.jar --server.port=8080
 ## How do I use it?
 
 - CITYdata is a REST API which receives queries as input and generates data as output.
-- A query is a JSON file where you specify which data you want and which transformations you wish to apply to the data. You can see query examples in the folder /docs/examples.
+- A query is a JSON file where you specify which data you want and which transformations you wish to apply to the data. You can see query examples in the folder `/docs/examples/queries`.
 - You can call CITYdata routes using your favourite programming language. For example, you can use the requests package in [Python](https://www.geeksforgeeks.org/get-post-requests-using-python/) or the fetch API in [JavaScript](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch).
 - If you are familiar with Postman, you can use our Postman collection [here](https://github.com/ptidejteam/citydata/blob/master/Middleware/docs/citydata_collection.json) to send your queries, no need to write code.
 
@@ -97,10 +119,59 @@ The following routes are available:
 
 For now, the number of Producers, Operations and parameters is quite limited, but we intend to expand it in the future and also document it better. Your suggestions are more than welcome!
 
+
+## Accessing Private/Protected Routes
+
+To access the private or protected routes, follow these steps
+
+- Create a User Account:
+	* Follow the instructions in the "Adding Users" section above to create a username and password
+
+- Authenticate the User:
+	* Run the application and send a `POST` request to the following endpoint
+			
+			http://localhost:8080/authenticate
+		
+Include your username and password in the request body as JSON, for example
+	
+```json
+{
+"username": "yourUsername",
+"password": "yourPassword"
+}
+```
+			
+- Receive Authentication Token: 
+	* Upon successful authentication, a token will be returned in the response
+
+- Copy the Token:
+	* Copy the token from the response. You'll use this to access protected routes
+
+- Access Protected Routes:
+	* Use one of the following endpoints:
+		
+		POST http://localhost:8080/apply/sync
+		POST http://localhost:8080/apply/async
+
+- Add Authorization Header in Postman:
+	* In Postman (or any API client)
+	* Go to the Authorization tab
+	* Set Type to Bearer Token
+	* Paste the token into the Token field
+
+- Access Granted:
+	* If the token is valid, Spring Boot will authorize your request and grant access to the protected route
+	
+## Available Data Sources
+
+- [Dataset Catalog](DATA_SOURCES.md) - Complete list of all available datasets
+
+
 ## Who do I talk to?
 
 Project manager: gabriel.cavalheiroullmann at concordia.ca
 
+
 ## Development Guidelines
 - The develop branch is the working branch for CityData middleware developers. To integrate your changes, please create a new branch based on develop, apply your changes, then open a **pull request** and set develop as a target.
 - One or more members of the CityData development team will **review every pull request** and provide improvement suggestions to the PR authors, if necessary.

Middleware/docs/examples/code/HttpAuthenticate.py

@@ -1,7 +1,7 @@
 import requests
 import json
 
-url = "http://localhost:8082/authenticate"
+url = "https://ngci.encs.concordia.ca/citydata/authenticate"
 
 payload = json.dumps({
   "username": "citydata",

Middleware/docs/examples/code/HttpGetAsync.java

@@ -12,7 +12,7 @@ public class HttpGetAsync {
 
 	public static void main(String[] args) throws Exception {
 		String runnerId = "d593c930-7fed-4c7b-ac52-fff946b78c32";
-		String requestUrl = "http://localhost:8082/apply/async/" + runnerId;
+		String requestUrl = "https://ngci.encs.concordia.ca/citydata/home" + runnerId;
 
 		URL url = new URL(requestUrl);
 		HttpURLConnection connection = (HttpURLConnection) url.openConnection();

Middleware/docs/examples/code/HttpGetAsyncWithJWT.java

@@ -0,0 +1,53 @@
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class HttpGetAsyncWithJWT {
+    
+    public static void main(String[] args) throws Exception {
+    	
+        String token = getJwtToken();
+        
+        // This runnerId is just an example; the user should use an actually existing runnerId from a Runner they created.
+        
+        String runnerId = "d593c930-7fed-4c7b-ac52-fff946b78c32";
+        String requestUrl = "http://localhost:8082/apply/async/" + runnerId;
+        
+        URL url = new URL(requestUrl);
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestMethod("GET");
+        connection.setRequestProperty("Authorization", "Bearer " + token);
+        
+        int responseCode = connection.getResponseCode();
+        System.out.println("GET Response Code: " + responseCode);
+        
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"))) {
+            StringBuilder response = new StringBuilder();
+            String responseLine;
+            while ((responseLine = br.readLine()) != null) {
+                response.append(responseLine.trim());
+            }
+            System.out.println(response.toString());
+        }
+    }
+    
+    private static String getJwtToken() throws Exception {
+        URL url = new URL("http://localhost:8082/authenticate");
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestMethod("POST");
+        connection.setRequestProperty("Content-Type", "application/json");
+        connection.setDoOutput(true);
+        
+        String json = "{\"username\":\"citydata\",\"password\":\"citydata\"}";
+        connection.getOutputStream().write(json.getBytes(StandardCharsets.UTF_8));
+        
+        BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+        return reader.readLine(); // JWT token returned directly as string
+    }
+}