Skip to content

Commit 9478f18

Browse files
alainabbasalainabbas
committed
initial
0 parents  commit 9478f18

File tree

13 files changed

+630
-0
lines changed

13 files changed

+630
-0
lines changed

‎.github/workflows/docker-image.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#
2+
name: Create and publish a Docker image
3+
4+
# Configures this workflow to run every time a change is pushed to the branch called `release`.
5+
on:
6+
release:
7+
types: [published]
8+
9+
env:
10+
REGISTRY: ghcr.io
11+
IMAGE_NAME: ${{ github.repository }}
12+
13+
jobs:
14+
build-and-push-image:
15+
runs-on: ubuntu-latest
16+
permissions:
17+
contents: read
18+
packages: write
19+
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
24+
- name: Log in to the Container registry
25+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
26+
with:
27+
registry: ${{ env.REGISTRY }}
28+
username: ${{ github.actor }}
29+
password: ${{ secrets.GITHUB_TOKEN }}
30+
31+
- name: Extract metadata (tags, labels) for Docker
32+
id: meta
33+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
34+
with:
35+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
36+
37+
- name: Build and push Docker image
38+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
39+
with:
40+
context: .
41+
push: true
42+
tags: ${{ steps.meta.outputs.tags }}
43+
labels: ${{ steps.meta.outputs.labels }}

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
cache/
2+
cache_old/
3+
logs/
4+
.env
5+
src/__pycache__/*
6+
.idea/
7+
lib/
8+
bin/
9+
pyvenv.cfg

‎Dockerfile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
ARG PYTHON_VERSION=3.11
2+
FROM python:${PYTHON_VERSION}-slim as base
3+
4+
ENV PYTHONDONTWRITEBYTECODE=1
5+
ENV PYTHONUNBUFFERED=1
6+
7+
WORKDIR /data
8+
9+
RUN mkdir data cache
10+
11+
ARG UID=10001
12+
RUN adduser \
13+
--disabled-password \
14+
--gecos "" \
15+
--home "/nonexistent" \
16+
--shell "/sbin/nologin" \
17+
--no-create-home \
18+
--uid "${UID}" \
19+
appuser
20+
21+
RUN chown -R appuser:appuser data cache
22+
23+
RUN --mount=type=cache,target=/root/.cache/pip \
24+
--mount=type=bind,source=requirements.txt,target=requirements.txt \
25+
python -m pip install -r requirements.txt
26+
27+
USER appuser
28+
29+
COPY . .
30+
31+
CMD python main.py

‎Makefile‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
include .env
2+
STC_API_TARGET?=193.52.197.92
3+
STC_API_HOST?=taiga.archi.fr
4+
STC_API_FORWARD_PORT?=1337
5+
IMG_NAME = "ghcr.io/libertech-fr/sesame-taiga_crawler"
6+
BASE_NAME = "sesame"
7+
APP_NAME = "sesame-taiga_crawler"
8+
PLATFORM = "linux/amd64"
9+
10+
.DEFAULT_GOAL := help
11+
help:
12+
@printf "\033[33mUsage:\033[0m\n make [target] [arg=\"val\"...]\n\n\033[33mTargets:\033[0m\n"
13+
@grep -E '^[-a-zA-Z0-9_\.\/]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) \
14+
| sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[32m%-15s\033[0m %s\n", $$1, $$2}'
15+
16+
build: ## Construit l'image docker
17+
@printf "\033[33mDOCKER:\033[0m Build docker image ...\n"
18+
@docker build -t sesame-taiga_crawler .
19+
@printf "\033[33mDOCKER:\033[0m SUCCESSFUL !!!\n"
20+
21+
pull-crawler-docker: ## Pull l'image docker
22+
@docker pull $(IMG_NAME)
23+
24+
run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
25+
@docker run --rm -it \
26+
--add-host host.docker.internal:host-gateway \
27+
--network dev \
28+
--platform $(PLATFORM) \
29+
--name $(APP_NAME) \
30+
-v $(CURDIR):/data \
31+
sesame-taiga_crawler
32+
33+
run-crawler: ## Lance le crawler Sesame - Taiga avec python !
34+
@python3 main.py
35+
36+
install-deps: ## Installe les dépendances python
37+
@printf "\033[33mPIP:\033[0m install required dependencies ...\n"
38+
@pip install -r requirements.txt
39+
@printf "\033[33mPIP:\033[0m SUCCESSFUL !!!\n"
40+
41+
taiga-forward: ## Transfert les appels de l'API Taiga via un proxy socks au travers du serveur sesame (à utiliser pour lancer le script à distance)
42+
@printf "\033[33mNCAT:\033[0m Launch forwarding tcp requests for <$(STC_API_HOST)> ...\n"
43+
@ssh libertech@$(STC_API_TARGET) "pkill -f 'ncat $(STC_API_HOST) 443'" || true
44+
@ssh libertech@$(STC_API_TARGET) "ncat --keep-open --no-shutdown -v \
45+
--sh-exec 'ncat $(STC_API_HOST) 443' \
46+
-l $(STC_API_FORWARD_PORT)"
47+
@printf "\033[33mNCAT:\033[0m End of forwarding requests !\n"
48+
49+
update-reqs: ## Met à jour la liste des dépendances python
50+
@printf "\033[33mUPDATE:\033[0m pipreqs dependency ...\n"
51+
@pip install pipreqs > ./logs/update-reqs.log
52+
@printf "\033[33mUPDATE:\033[0m requirements.txt ...\n"
53+
python -m pipreqs.pipreqs --force .
54+
@printf "\033[33mUPDATE:\033[0m SUCCESSFUL !!!\n"

‎README.md‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Sesame - Taiga Crawler
2+
3+
Le Sesame - Taiga Crawler est un projet Python qui interagit avec l'API Taiga pour effectuer des tâches spécifiques.
4+
5+
## Table des matières
6+
7+
- [Sesame - Taiga Crawler](#sesame---taiga-crawler)
8+
- [Table des matières](#table-des-matières)
9+
- [Aperçu](#aperçu)
10+
- [Installation](#installation)
11+
- [Configuration](#configuration)
12+
- [Utilisation](#utilisation)
13+
- [Développement](#développement)
14+
15+
## Aperçu
16+
17+
Le Sesame - Taiga Crawler est conçu pour effectuer des tâches liées à l'API Taiga. Il comprend des fonctionnalités telles que la vérification AmoinsB et l'exportation de données individuelles. Le projet est structuré avec des composants modulaires et utilise la programmation asynchrone pour l'efficacité.
18+
19+
## Installation
20+
21+
1. Requis :
22+
- Python 3
23+
- PIP
24+
25+
2. Clonez le dépôt :
26+
27+
```bash
28+
git clone https://github.com/libertech-fr/sesame-taiga_crawler.git
29+
cd taiga-crawler
30+
```
31+
32+
3. Installez les dépendances :
33+
34+
```bash
35+
make install-deps
36+
```
37+
38+
## Configuration
39+
40+
1. Créez un fichier `.env` à la racine du projet et ajoutez la configuration suivante :
41+
42+
```dotenv
43+
STC_API_BASEURL=https://[URL_DU_SERVEUR_AUTHORISE]:1337
44+
STC_API_USERNAME=[IDENTIFIANT_DE_L_API]
45+
STC_API_PASSWORD=[PASSWORD_DE_L_API]
46+
STC_API_FORWARD_PORT=1337
47+
STC_API_PASSENSA=[PASSWORD_ENSA]
48+
49+
SESAME_API_BASEURL=http://[URL_DU_SERVEUR_AUTHORISE]:4002
50+
```
51+
52+
Assurez-vous de remplacer les valeurs fictives par vos informations réelles.
53+
54+
- **STC_API_BASEURL** : Peut être vide si vous lancez le logiciel depuis le serveur autorisé
55+
- **STC_API_FORWARD_PORT** : Non requis si `make taiga-forward` n'est pas utilisé
56+
## Utilisation
57+
58+
1. Mise en route
59+
60+
!!! ATTENTION !!!
61+
Vous êtes sur la machine authorisée par L'API Taiga : Alors ignorez cette étape !!!
62+
63+
```bash
64+
make taiga-forward
65+
```
66+
67+
2. Exécution du script
68+
69+
Exécutez le script principal :
70+
71+
```bash
72+
make run-crawler
73+
```
74+
75+
## Développement
76+
77+
Vous pouvez utiliser Pycharm avec la Makefile extension, le débuggeur est automatiquement configuré !
78+
79+
- Mise à jour des requirements :
80+
```bash
81+
make update-reqs
82+
```

‎config.yml‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
---
2+
taiga_etd.json:
3+
mapping:
4+
inetOrgPerson.cn:
5+
- "nom"
6+
- "prenom"
7+
inetOrgPerson.displayName:
8+
- "prenom"
9+
- "nom"
10+
inetOrgPerson.sn: "nom"
11+
inetOrgPerson.givenName: "prenom"
12+
additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
13+
inetOrgPerson.employeeNumber: "id_coord"
14+
"$setOnInsert.inetOrgPerson.uid":
15+
- "prenom"
16+
- "nom"
17+
additionalFields.attributes.supannPerson.supannRefId: "id_coord"
18+
"$setOnInsert.inetOrgPerson.mail":
19+
- "prenom"
20+
- "nom"
21+
inetOrgPerson.mobile: "tel_mob"
22+
inetOrgPerson.postalAddress:
23+
- "adresse"
24+
- "CP"
25+
- "Ville"
26+
additionalFields.attributes.supannPerson.supanncivilite: "civilite"
27+
additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
28+
additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
29+
additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
30+
additionalFields.attributes.supannPerson.supannAutreMail: "email2"
31+
additionalFields:
32+
additionalFields.objectClasses:
33+
- "supannPerson"
34+
additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "etd"
35+
state: -1
36+
inetOrgPerson.employeeType: "TAIGA"
37+
inetOrgPerson.departmentNumber: "etd"
38+
additionalFields.attributes.supannPerson.supannEtablissement: "{UAI}0690184A"
39+
transforms:
40+
inetOrgPerson.cn: "join(delimiter=' ')"
41+
inetOrgPerson.displayName: "join(delimiter=' ')"
42+
inetOrgPerson.postalAddress: "join(delimiter=',')"
43+
"$setOnInsert.inetOrgPerson.uid":
44+
- "join(delimiter='.')"
45+
- "remove_accents"
46+
- "lower"
47+
- "replace(old=' ',new='-')"
48+
# - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
49+
"$setOnInsert.inetOrgPerson.mail":
50+
- "join(delimiter='.')"
51+
- "lower"
52+
- "remove_accents"
53+
- "replace(old=' ',new='-')"
54+
- "suffix(string='@sesame.local')"
55+
# taiga_adm.json:
56+
# mapping:
57+
# inetOrgPerson.cn:
58+
# - "nom"
59+
# - "prenom"
60+
# inetOrgPerson.displayName:
61+
# - "prenom"
62+
# - "nom"
63+
# inetOrgPerson.sn: "nom"
64+
# inetOrgPerson.givenName: "prenom"
65+
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
66+
# inetOrgPerson.employeeNumber: "id_coord"
67+
# $setOnInsert.inetOrgPerson.uid:
68+
# - "prenom"
69+
# - "nom"
70+
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
71+
# $setOnInsert.inetOrgPerson.mail:
72+
# - "prenom"
73+
# - "nom"
74+
# inetOrgPerson.mobile: "tel_mob"
75+
# inetOrgPerson.postalAddress:
76+
# - "adresse"
77+
# - "CP"
78+
# - "Ville"
79+
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
80+
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
81+
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
82+
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
83+
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
84+
# additionalFields:
85+
# additionalFields.objectClasses:
86+
# - "supannPerson"
87+
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "adm"
88+
# inetOrgPerson.employeeType: "TAIGA"
89+
# inetOrgPerson.departmentNumber: "adm"
90+
# state: -1
91+
# transforms:
92+
# inetOrgPerson.cn: "join(delimiter=' ')"
93+
# inetOrgPerson.displayName: "join(delimiter=' ')"
94+
# inetOrgPerson.postalAddress: "join(delimiter=',')"
95+
# $setOnInsert.inetOrgPerson.uid:
96+
# - "join(delimiter='.')"
97+
# - "remove_accents"
98+
# - "lower"
99+
# - "replace(old=' ',new='-')"
100+
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
101+
# $setOnInsert.inetOrgPerson.mail:
102+
# - "join(delimiter='.')"
103+
# - "remove_accents"
104+
# - "lower"
105+
# - "replace(old=' ',new='-')"
106+
# - "suffix(string='@sesame.local')"
107+
# taiga_esn.json:
108+
# mapping:
109+
# inetOrgPerson.cn:
110+
# - "nom"
111+
# - "prenom"
112+
# inetOrgPerson.displayName:
113+
# - "prenom"
114+
# - "nom"
115+
# inetOrgPerson.sn: "nom"
116+
# inetOrgPerson.givenName: "prenom"
117+
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
118+
# inetOrgPerson.employeeNumber: "id_coord"
119+
# $setOnInsert.inetOrgPerson.uid:
120+
# - "prenom"
121+
# - "nom"
122+
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
123+
# $setOnInsert.inetOrgPerson.mail:
124+
# - "prenom"
125+
# - "nom"
126+
# inetOrgPerson.mobile: "tel_mob"
127+
# inetOrgPerson.postalAddress:
128+
# - "adresse"
129+
# - "CP"
130+
# - "Ville"
131+
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
132+
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
133+
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
134+
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
135+
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
136+
# additionalFields:
137+
# additionalFields.objectClasses:
138+
# - "supannPerson"
139+
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "esn"
140+
# inetOrgPerson.departmentNumber: "esn"
141+
# inetOrgPerson.employeeType: "TAIGA"
142+
# state: -1
143+
# transforms:
144+
# inetOrgPerson.cn: "join(delimiter=' ')"
145+
# inetOrgPerson.displayName: "join(delimiter=' ')"
146+
# inetOrgPerson.postalAddress: "join(delimiter=',')"
147+
# $setOnInsert.inetOrgPerson.uid:
148+
# - "join(delimiter='.')"
149+
# - "remove_accents"
150+
# - "lower"
151+
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
152+
# $setOnInsert.inetOrgPerson.mail:
153+
# - "join(delimiter='.')"
154+
# - "remove_accents"
155+
# - "lower"
156+
# - "replace(old=' ',new='-')"
157+
# - "suffix(string='@sesame.local')"

‎data/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

0 commit comments

Comments
 (0)