forked from dataforgoodfr/12_bloom
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathapp.py
More file actions
53 lines (43 loc) · 1.63 KB
/
app.py
File metadata and controls
53 lines (43 loc) · 1.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
"""Bloom scrapper.
This application execute the scrapping of different vessel's positions source,
based on a list of vessel. It can run in local mode, with a custom scheduler
or scheduled outside of the app by a cronjob.
"""
import argparse
from logging import getLogger
from bloom.enums import ExecutionMode
from bloom.infra.marine_traffic_scraper import MarineTrafficVesselScraper
from bloom.infra.repository_vessel import VesselRepository
from bloom.scheduler import PeriodicScheduler
from bloom.usecase.ScrapVesselsFromMarineTraffic import ScrapVesselsFromMarineTraffic
SCRAP_INTERVAL = 15 * 60
logger = getLogger()
def scrap_vessels_with_marine_traffic() -> None:
vessels_repository = VesselRepository()
scraper = MarineTrafficVesselScraper()
ScrapVesselsFromMarineTraffic(vessels_repository, scraper).scrap_vessels()
def main() -> None:
parser = argparse.ArgumentParser(description="Bloom scraping application")
parser.add_argument(
"-m",
"--mode",
type=ExecutionMode,
help="execution mode of the scraper",
required=False,
default=ExecutionMode.CRONTAB,
)
args = parser.parse_args()
if args.mode == ExecutionMode.LOCAL:
logger.info("Starting scraping with internal scheduler")
scheduler = PeriodicScheduler(
function=scrap_vessels_with_marine_traffic,
interval=SCRAP_INTERVAL,
)
scrap_vessels_with_marine_traffic()
while True:
scheduler.start()
else:
logger.info("Starting scraping with external scheduler")
scrap_vessels_with_marine_traffic()
if __name__ == "__main__":
main()