From 3eb99ae7ae58fd7cb83f558d175febe8b40cca1a Mon Sep 17 00:00:00 2001 From: Sviatoslav Tsariov Date: Fri, 15 Sep 2023 13:16:14 +0300 Subject: [PATCH] director added --- director/.dockerignore | 52 +++++ director/.env.example | 3 + director/Dockerfile | 12 + director/app.py | 31 +++ director/config.py | 18 ++ director/director.py | 66 ++++++ director/docker-compose.yaml | 11 + director/requirements.txt | 10 + director/utils/libs/decouple.py | 314 ++++++++++++++++++++++++++ director/utils/libs/loggedRequests.py | 264 ++++++++++++++++++++++ director/utils/logger.py | 69 ++++++ director/utils/utils.py | 19 ++ director/wrapper.sh | 27 +++ parser/collect.py | 4 +- 14 files changed, 898 insertions(+), 2 deletions(-) create mode 100644 director/.dockerignore create mode 100644 director/.env.example create mode 100644 director/Dockerfile create mode 100644 director/app.py create mode 100644 director/config.py create mode 100644 director/director.py create mode 100644 director/docker-compose.yaml create mode 100644 director/requirements.txt create mode 100644 director/utils/libs/decouple.py create mode 100644 director/utils/libs/loggedRequests.py create mode 100644 director/utils/logger.py create mode 100644 director/utils/utils.py create mode 100755 director/wrapper.sh diff --git a/director/.dockerignore b/director/.dockerignore new file mode 100644 index 0000000..7513fb2 --- /dev/null +++ b/director/.dockerignore @@ -0,0 +1,52 @@ +# Ignore common Python-related files and directories +__pycache__ +*.pyc +*.pyo +*.pyd + +# Ignore development and version control files +*.log +*.swp +.git +.vscode +.idea +__init__.py + +# Ignore virtual environment and dependencies +venv/ +env/ +*.egg-info/ +*.egg +*.log + +# Ignore other temporary and build files +*.bak +*.bak1 +*.bak2 +*.swp +*.swo +*.swn +*.un~ +*.swm +*.swl +*.pyc +*.pyo +*.bak +*.bak1 +*.bak2 +*.orig +*.rej +*.log +*.backup +*.tmp +*.tmp.* +*.diff +*.cache +*.css.map +*.css.map.* + +# Ignore any specific files or directories your project generates +# Add lines for any other project-specific files or directories to ignore +.env +.env* +*.env diff --git a/director/.env.example b/director/.env.example new file mode 100644 index 0000000..14032b9 --- /dev/null +++ b/director/.env.example @@ -0,0 +1,3 @@ +DOWNLOADER_LINK= +PARSER_ENDPOINT_LIST= +SCHEDULE_TIME= diff --git a/director/Dockerfile b/director/Dockerfile new file mode 100644 index 0000000..5ae87ac --- /dev/null +++ b/director/Dockerfile @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 + +FROM python:3.8-slim-buster + +WORKDIR /app + +RUN apt-get update && \ + rm -rf /var/lib/apt/lists/* + +COPY . . + +ENTRYPOINT ["/app/wrapper.sh"] diff --git a/director/app.py b/director/app.py new file mode 100644 index 0000000..bfd68c3 --- /dev/null +++ b/director/app.py @@ -0,0 +1,31 @@ +import asyncio +import aiohttp +import schedule +import time +import random +import threading + +from config import App + +def main(): + print("Script is running...") + +def run_scheduler(): + while True: + schedule.run_pending() + time.sleep(1) + +if __name__ == '__main__': + # Schedule the script to run every day at a specific time + print(f'Repos downloading is scheduled on {App.ScheduleTime}') + schedule.every().day.at(App.ScheduleTime).do(lambda: main()) + + # Run the scheduler in a separate thread + scheduler_thread = threading.Thread(target=run_scheduler) + scheduler_thread.start() + + # Keep the main thread running + try: + scheduler_thread.join() + except KeyboardInterrupt: + pass diff --git a/director/config.py b/director/config.py new file mode 100644 index 0000000..b2bf91e --- /dev/null +++ b/director/config.py @@ -0,0 +1,18 @@ +from utils.libs.decouple import config, UndefinedValueError +from utils.utils import str_to_bool +from utils.logger import logger + +def set_conf_value(key): + try: + value = config(key) + if value == '': + logger.warn(f'WARNING: The variable {key} is an empty string.') + return value + except (UndefinedValueError): + logger.warn(f'WARNING: Please set the variable {key} in the .env file based on .env.example.') + return None + +class App: + DownloaderLink=set_conf_value('DOWNLOADER_LINK') + ParserEndpointList=set_conf_value('PARSER_ENDPOINT_LIST') + ScheduleTime=set_conf_value('SCHEDULE_TIME') diff --git a/director/director.py b/director/director.py new file mode 100644 index 0000000..8ab6bff --- /dev/null +++ b/director/director.py @@ -0,0 +1,66 @@ +import asyncio +import aiohttp +import schedule +import time +import random +import threading + +from config import App + +async def fetch_endpoint(session, url): + async with session.get(url) as response: + return await response.text() + +async def fetch_endpoint_with_retry(session, url): + timeout = aiohttp.ClientTimeout(total=None) # No timeout + + async with session.get(url, timeout=timeout) as response: + if response.status == 200: + return await response.text() + elif response.status == 500: + retry_delay = random.randint(30, 90) + print(f"Received a 500 response from {url}. Retrying in {retry_delay} seconds.") + await asyncio.sleep(retry_delay) + return await fetch_endpoint_with_retry(session, url) + else: + print(f"Received an unexpected response code {response.status} from {url}.") + return None + + +async def main(): + print("Script is running...") + + async with aiohttp.ClientSession() as session: + # Fetch all endpoints concurrently + endpoints = App.ParserEndpointList.split() + + # Create tasks for fetching all endpoints concurrently + tasks = [fetch_endpoint_with_retry(session, url) for url in endpoints] + + # Wait for all endpoint tasks to complete + endpoint_responses = await asyncio.gather(*tasks) + print(endpoint_responses) + + # Check if all endpoints returned 200 status codes + if all(response is not None for response in endpoint_responses): + # Call another_endpoint only if all endpoints returned 200 + another_endpoint = App.DownloaderLink + response = await fetch_endpoint(session, another_endpoint) + # Process the response from the final endpoint + print(response) + else: + print("Not all endpoints returned 200 status codes. Skipping another_endpoint.") + +def run_scheduler(): + while True: + schedule.run_pending() + time.sleep(1) + +if __name__ == '__main__': + # Schedule the script to run every day at a specific time + print(f'Repos downloading is scheduled on {App.ScheduleTime}') + schedule.every().day.at(App.ScheduleTime).do(lambda: asyncio.run(main())) + + # Run the scheduler in a separate thread + scheduler_thread = threading.Thread(target=run_scheduler) + scheduler_thread.start() diff --git a/director/docker-compose.yaml b/director/docker-compose.yaml new file mode 100644 index 0000000..6d4bedf --- /dev/null +++ b/director/docker-compose.yaml @@ -0,0 +1,11 @@ +version: '2.1' +services: + director: + container_name: director + image: director + build: . + network_mode: host + environment: + DOWNLOADER_LINK: ${DOWNLOADER_LINK} + PARSER_ENDPOINT_LIST: ${PARSER_ENDPOINT_LIST} + SCHEDULE_TIME: ${SCHEDULE_TIME} diff --git a/director/requirements.txt b/director/requirements.txt new file mode 100644 index 0000000..dc7c1f4 --- /dev/null +++ b/director/requirements.txt @@ -0,0 +1,10 @@ +aiohttp==3.8.5 +aiosignal==1.3.1 +async-timeout==4.0.3 +attrs==23.1.0 +charset-normalizer==3.2.0 +frozenlist==1.4.0 +idna==3.4 +multidict==6.0.4 +schedule==1.2.0 +yarl==1.9.2 diff --git a/director/utils/libs/decouple.py b/director/utils/libs/decouple.py new file mode 100644 index 0000000..f05abd4 --- /dev/null +++ b/director/utils/libs/decouple.py @@ -0,0 +1,314 @@ +# coding: utf-8 +import os +import sys +import string +from shlex import shlex +from io import open +from collections import OrderedDict + +# Useful for very coarse version differentiation. +PYVERSION = sys.version_info + +if PYVERSION >= (3, 0, 0): + from configparser import ConfigParser + text_type = str +else: + from ConfigParser import SafeConfigParser as ConfigParser + text_type = unicode + +if PYVERSION >= (3, 2, 0): + def read_config(parser, file): return parser.read_file(file) +else: + def read_config(parser, file): return parser.readfp(file) + + +DEFAULT_ENCODING = 'UTF-8' + + +# Python 3.10 don't have strtobool anymore. So we move it here. +TRUE_VALUES = {"y", "yes", "t", "true", "on", "1"} +FALSE_VALUES = {"n", "no", "f", "false", "off", "0"} + +def strtobool(value): + if isinstance(value, bool): + return value + value = value.lower() + + if value in TRUE_VALUES: + return True + elif value in FALSE_VALUES: + return False + + raise ValueError("Invalid truth value: " + value) + + +class UndefinedValueError(Exception): + pass + + +class Undefined(object): + """ + Class to represent undefined type. + """ + pass + + +# Reference instance to represent undefined values +undefined = Undefined() + + +class Config(object): + """ + Handle .env file format used by Foreman. + """ + + def __init__(self, repository): + self.repository = repository + + def _cast_boolean(self, value): + """ + Helper to convert config values to boolean as ConfigParser do. + """ + value = str(value) + return bool(value) if value == '' else bool(strtobool(value)) + + @staticmethod + def _cast_do_nothing(value): + return value + + def get(self, option, default=undefined, cast=undefined): + """ + Return the value for option or default if defined. + """ + + # We can't avoid __contains__ because value may be empty. + if option in os.environ: + value = os.environ[option] + elif option in self.repository: + value = self.repository[option] + else: + if isinstance(default, Undefined): + raise UndefinedValueError( + '{} not found. Declare it as envvar or define a default value.'.format(option)) + + value = default + + if isinstance(cast, Undefined): + cast = self._cast_do_nothing + elif cast is bool: + cast = self._cast_boolean + + return cast(value) + + def __call__(self, *args, **kwargs): + """ + Convenient shortcut to get. + """ + return self.get(*args, **kwargs) + + +class RepositoryEmpty(object): + def __init__(self, source='', encoding=DEFAULT_ENCODING): + pass + + def __contains__(self, key): + return False + + def __getitem__(self, key): + return None + + +class RepositoryIni(RepositoryEmpty): + """ + Retrieves option keys from .ini files. + """ + SECTION = 'settings' + + def __init__(self, source, encoding=DEFAULT_ENCODING): + self.parser = ConfigParser() + with open(source, encoding=encoding) as file_: + read_config(self.parser, file_) + + def __contains__(self, key): + return (key in os.environ or + self.parser.has_option(self.SECTION, key)) + + def __getitem__(self, key): + return self.parser.get(self.SECTION, key) + + +class RepositoryEnv(RepositoryEmpty): + """ + Retrieves option keys from .env files with fall back to os.environ. + """ + + def __init__(self, source, encoding=DEFAULT_ENCODING): + self.data = {} + + with open(source, encoding=encoding) as file_: + for line in file_: + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + k, v = line.split('=', 1) + k = k.strip() + v = v.strip() + if len(v) >= 2 and ((v[0] == "'" and v[-1] == "'") or (v[0] == '"' and v[-1] == '"')): + v = v[1:-1] + self.data[k] = v + + def __contains__(self, key): + return key in os.environ or key in self.data + + def __getitem__(self, key): + return self.data[key] + + +class RepositorySecret(RepositoryEmpty): + """ + Retrieves option keys from files, + where title of file is a key, content of file is a value + e.g. Docker swarm secrets + """ + + def __init__(self, source='/run/secrets/'): + self.data = {} + + ls = os.listdir(source) + for file in ls: + with open(os.path.join(source, file), 'r') as f: + self.data[file] = f.read() + + def __contains__(self, key): + return key in os.environ or key in self.data + + def __getitem__(self, key): + return self.data[key] + + +class AutoConfig(object): + """ + Autodetects the config file and type. + + Parameters + ---------- + search_path : str, optional + Initial search path. If empty, the default search path is the + caller's path. + + """ + SUPPORTED = OrderedDict([ + ('settings.ini', RepositoryIni), + ('.env', RepositoryEnv), + ]) + + encoding = DEFAULT_ENCODING + + def __init__(self, search_path=None): + self.search_path = search_path + self.config = None + + def _find_file(self, path): + # look for all files in the current path + for configfile in self.SUPPORTED: + filename = os.path.join(path, configfile) + if os.path.isfile(filename): + return filename + + # search the parent + parent = os.path.dirname(path) + if parent and parent != os.path.abspath(os.sep): + return self._find_file(parent) + + # reached root without finding any files. + return '' + + def _load(self, path): + # Avoid unintended permission errors + try: + filename = self._find_file(os.path.abspath(path)) + except Exception: + filename = '' + Repository = self.SUPPORTED.get(os.path.basename(filename), RepositoryEmpty) + + self.config = Config(Repository(filename, encoding=self.encoding)) + + def _caller_path(self): + # MAGIC! Get the caller's module path. + frame = sys._getframe() + path = os.path.dirname(frame.f_back.f_back.f_code.co_filename) + return path + + def __call__(self, *args, **kwargs): + if not self.config: + self._load(self.search_path or self._caller_path()) + + return self.config(*args, **kwargs) + + +# A pré-instantiated AutoConfig to improve decouple's usability +# now just import config and start using with no configuration. +config = AutoConfig() + +# Helpers + +class Csv(object): + """ + Produces a csv parser that return a list of transformed elements. + """ + + def __init__(self, cast=text_type, delimiter=',', strip=string.whitespace, post_process=list): + """ + Parameters: + cast -- callable that transforms the item just before it's added to the list. + delimiter -- string of delimiters chars passed to shlex. + strip -- string of non-relevant characters to be passed to str.strip after the split. + post_process -- callable to post process all casted values. Default is `list`. + """ + self.cast = cast + self.delimiter = delimiter + self.strip = strip + self.post_process = post_process + + def __call__(self, value): + """The actual transformation""" + if value is None: + return self.post_process() + + def transform(s): return self.cast(s.strip(self.strip)) + + splitter = shlex(value, posix=True) + splitter.whitespace = self.delimiter + splitter.whitespace_split = True + + return self.post_process(transform(s) for s in splitter) + + +class Choices(object): + """ + Allows for cast and validation based on a list of choices. + """ + + def __init__(self, flat=None, cast=text_type, choices=None): + """ + Parameters: + flat -- a flat list of valid choices. + cast -- callable that transforms value before validation. + choices -- tuple of Django-like choices. + """ + self.flat = flat or [] + self.cast = cast + self.choices = choices or [] + + self._valid_values = [] + self._valid_values.extend(self.flat) + self._valid_values.extend([value for value, _ in self.choices]) + + def __call__(self, value): + transform = self.cast(value) + if transform not in self._valid_values: + raise ValueError(( + 'Value not in list: {!r}; valid values are {!r}' + ).format(value, self._valid_values)) + else: + return transform diff --git a/director/utils/libs/loggedRequests.py b/director/utils/libs/loggedRequests.py new file mode 100644 index 0000000..4904072 --- /dev/null +++ b/director/utils/libs/loggedRequests.py @@ -0,0 +1,264 @@ +import re +import requests +from http import HTTPStatus +import json as JSON + +from infra.config import BaseUri, Settings +from utils.logger import logger, bcolors +from utils.singleton import SingletonMeta + +swagger_api_json_endpoint = '/api-json' +api_info_urls = { + BaseUri.Iguana: 'Iguana', + BaseUri.Pyrador: 'Pyrador', + BaseUri.Zoo: 'Zoo' +} +excluded_endpoints = [ + ('POST', BaseUri.Iguana + '/api/v1/test/controller'), + ('GET', BaseUri.Iguana + '/api/v1/test/controller'), + ('DELETE', BaseUri.Iguana + '/api/v1/test/controller'), + ('GET', BaseUri.Iguana + '/api/v1/health'), + ('GET', BaseUri.Iguana + '/api/v1/metrics'), + ('GET', BaseUri.Iguana + '/api/v1/settings'), + ('POST', BaseUri.Iguana + '/api/v1/settings'), + ('PUT', BaseUri.Iguana + '/api/v1/settings'), + ('GET', BaseUri.Iguana + '/api/v1/activity'), + ('GET', BaseUri.Iguana + '/api/v1/activity/{activity_id}'), + ('POST', BaseUri.Iguana + '/api/v1/doorlock'), + ('PUT', BaseUri.Iguana + '/api/v1/profile/set-account-number'), + ('PUT', BaseUri.Iguana + '/api/v1/profile/address'), + ('PUT', BaseUri.Iguana + '/api/v1/profile/contact'), + ('POST', BaseUri.Iguana + '/api/v1/profile/set-firebase-token'), + ('PUT', BaseUri.Iguana + '/api/v1/profile/balance'), + ('GET', BaseUri.Iguana + '/api/v1/providable-service/{place_id}'), + ('POST', BaseUri.Iguana + '/api/v1/light-device/toggle'), + ('GET', BaseUri.Iguana + '/api/v1/light-device/state/{device_id}'), + ('PUT', BaseUri.Iguana + '/api/v1/user-place/{place_id}'), + ('GET', BaseUri.Iguana + '/api/v1/user-place/{place_id}/services'), + ('PUT', BaseUri.Iguana + '/api/v1/user-place/set/status'), + ('POST', BaseUri.Iguana + '/api/v1/profile/device/to/service'), + ('DELETE', BaseUri.Iguana + '/api/v1/profile/device/from/service'), + ('GET', BaseUri.Iguana + '/api/v1/profile/place/{place_id}/service/devices/{device_category}'), + ('POST', BaseUri.Iguana + '/api/v1/room'), + ('GET', BaseUri.Iguana + '/api/v1/room/by/place/{parent_id}'), + ('PUT', BaseUri.Iguana + '/api/v1/room/{id}'), + ('DELETE', BaseUri.Iguana + '/api/v1/room/{id}'), + ('GET', BaseUri.Iguana + '/api/v1/device/list/{type}'), + ('DELETE', BaseUri.Iguana + '/api/v1/user-place/qrcode'), + ('POST', BaseUri.Iguana + '/api/v1/billing'), + ('POST', BaseUri.Iguana + '/api/v1/intercom/acceptCall'), # TODO: test it with notifications + ('POST', BaseUri.Iguana + '/api/v1/upload/avatar'), # TODO: unable to test + ('POST', BaseUri.Zoo + '/api/v1/notifications/send-notification'), + ('POST', BaseUri.Zoo + '/api/v1/notifications/send-sms'), + ('DELETE', BaseUri.Zoo + '/api/v1/place/available_services') +] + +class APICoverageTracker(metaclass=SingletonMeta): + def __init__(self): + self.called_endpoints = {} + self.api_info = self.request_api_info(api_info_urls) + self.build_called_endpoints() + + def request_api_info(self, urls): + api_info = {} + + for url in urls: + res = requests.get(url + swagger_api_json_endpoint) + api_info[url] = res.json() + + return api_info + + def build_called_endpoints(self): + for url, info in self.api_info.items(): + try: + paths = info.get('paths') + if not url in self.called_endpoints: + self.called_endpoints[url] = {} + for path, methods in paths.items(): + endpoint = url + path + self.called_endpoints[url][path] = {} + for method, method_info in methods.items(): + if (method.upper(), endpoint) in excluded_endpoints: + continue + self.called_endpoints[url][path][method] = 0 + except Exception as e: + logger.error('Error happened while getting api info:', e) + + def endpoint_is_called(self, called_endpoint, method): + if not Settings.EnableCoverageStatistics: + return + + for url, paths in self.called_endpoints.items(): + for path, methods in paths.items(): + endpoint = url + path + pattern = re.sub(r'{.+?}', r'[^/]+', endpoint) + '$' + if re.match(pattern, called_endpoint) and method.lower() in methods: + self.called_endpoints[url][path][method.lower()] += 1 + return + + def print_coverage(self): + def calculate_coverage_statistics(total_urls, covered_urls): + if total_urls == 0: + return 0 + coverage_percentage = int(covered_urls / total_urls * 100) + + if coverage_percentage < 50: + color = bcolors.FAIL + elif coverage_percentage < 75: + color = bcolors.WARNING + else: + color = bcolors.OKGREEN + + statistics = f'{coverage_percentage}% ({covered_urls} / {total_urls})' + return f'{color}{statistics}{bcolors.ENDC}' + + def count_urls(gateway_url): + urls_num = 0 + covered_urls_num = 0 + + for url, paths in self.called_endpoints.items(): + for path, methods in paths.items(): + endpoint = url + path + if gateway_url in endpoint: + for method, num_of_calls in methods.items(): + urls_num += 1 + if num_of_calls > 0: + covered_urls_num += 1 + else: + logger.warn(f'{method.upper()} {endpoint} is not covered') + + return urls_num, covered_urls_num + + if not Settings.EnableCoverageStatistics: + return + + urls_num_sum = 0 + covered_urls_num_sum = 0 + + urls_info = \ + [(gateway_name, count_urls(gateway_url)) \ + for gateway_url, gateway_name in api_info_urls.items()] + + logger.info('Coverage statistics:') + logger.info() + + for gateway_name, (urls_num, covered_urls_num) in urls_info: + coverage_statistics = calculate_coverage_statistics(urls_num, covered_urls_num) + message = f' {gateway_name}: {coverage_statistics}' + logger.info(message) + + urls_num_sum += urls_num + covered_urls_num_sum += covered_urls_num + + coverage_statistics = \ + calculate_coverage_statistics(urls_num_sum, covered_urls_num_sum) + logger.info() + logger.info(f' Total: {coverage_statistics}\n') + +class Response(requests.Response): + def __init__(self, status_code=HTTPStatus.OK): + super().__init__() + self.status_code = status_code + +def log_req(method, url, params=None, data=None, json=None, headers=None): + logger.verbose(f'============================================================') + logger.verbose(f'[REQUEST] {method} {url}') + if params: + logger.verbose(f'params: {params}') + if data: + data = JSON.dumps(data, sort_keys=True, indent=4) + logger.verbose(f'data: {data}') + if json: + json = JSON.dumps(json, sort_keys=True, indent=4) + logger.verbose(f'json: {json}') + if headers: + headers = JSON.dumps(headers, sort_keys=True, indent=4) + logger.verbose(f'headers: {headers}') + +def log_res(res: requests.Response): + req = res.request + logger.verbose(f'[RESPONSE] {req.method} {req.url} {res.status_code}') + try: + json = JSON.dumps(res.json(), sort_keys=True, indent=4).replace('\\"', '"') + lines_num = json.count('\n') + max_lines_num = Settings.LoggingResponseMaxLinesNum + if lines_num <= max_lines_num: + logger.verbose(f'json: {json}') + else: + stats = f'{lines_num}/{max_lines_num}' + logger.verbose(f'Maximum number of lines for response exceeded:', stats) + except ValueError: + logger.verbose('response:', res.content) + except Exception as e: + logger.verbose(e) + +def request(method, url, headers=None, **kwargs): + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=kwargs.get('data'), json=kwargs.get('json'), headers=headers) + res = requests.request(method, url, **kwargs) + log_res(res) + return res + +def get(url, params=None, headers=None, **kwargs): + method = 'GET' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=params, \ + data=kwargs.get('data'), json=kwargs.get('json'), headers=headers) + res = requests.get(url, params=params, headers=headers, **kwargs) + log_res(res) + return res + +def options(url, headers=None, **kwargs): + method = 'OPTIONS' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=kwargs.get('data'), json=kwargs.get('json'), headers=headers) + res = requests.options(url, headers=headers, **kwargs) + log_res(res) + return res + +def head(url, headers=None, **kwargs): + method = 'HEAD' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=kwargs.get('data'), json=kwargs.get('json'), headers=headers) + res = requests.head(url, headers=headers, **kwargs) + log_res(res) + return res + +def post(url, data=None, json=None, headers=None, **kwargs): + method = 'POST' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=data, json=json, headers=headers) + res = requests.post(url, data=data, json=json, headers=headers, **kwargs) + log_res(res) + return res + +def put(url, data=None, headers=None, **kwargs): + method = 'PUT' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=data, json=kwargs.get('json'), headers=headers), + res = requests.put(url, data=data, headers=headers, **kwargs) + log_res(res) + return res + +def patch(url, data=None, headers=None, **kwargs): + method = 'PATCH' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=data, json=kwargs.get('json'), headers=headers) + res = requests.patch(url, data=data, headers=headers, **kwargs) + log_res(res) + return res + +def delete(url, headers=None, **kwargs): + method = 'DELETE' + APICoverageTracker().endpoint_is_called(url, method) + log_req(method, url, params=kwargs.get('params'), \ + data=kwargs.get('data'), json=kwargs.get('json'), headers=headers) + res = requests.delete(url, headers=headers, **kwargs) + log_res(res) + return res \ No newline at end of file diff --git a/director/utils/logger.py b/director/utils/logger.py new file mode 100644 index 0000000..3f4c473 --- /dev/null +++ b/director/utils/logger.py @@ -0,0 +1,69 @@ +import inspect +import traceback +from utils.utils import trace_origin + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +class logger: + def debug(*args, **kwargs): + from infra.config import Settings + if not Settings.EnableDebugMessages: + return + print(bcolors.HEADER, end='[DEBUG] ') + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + def error(*args, **kwargs): + print(bcolors.FAIL, end='[ERROR] ') + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + def warn(*args, trace_origin_flag=False, stacklevel=0, \ + print_stack=False, **kwargs): + + print(bcolors.WARNING, end='[WARNING] ') + if trace_origin_flag: + trace_origin(inspect.currentframe(), stacklevel) + if print_stack: + print(traceback.format_exc()) + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + def verbose(*args, trace_origin_flag=False, stacklevel=0, **kwargs): + from infra.config import Settings + if not Settings.EnableVerboseMessages: + return + if trace_origin_flag: + trace_origin(inspect.currentframe(), stacklevel) + print(bcolors.OKCYAN, end='[VERBOSE] ') + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + def log(*args, **kwargs): + print(bcolors.OKGREEN, end='[LOG] ') + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + def info(*args, **kwargs): + print(bcolors.OKBLUE, end='[INFO] ') + print(*args, **kwargs) + print(bcolors.ENDC, end='') + + class DisableVerbose(object): + def __enter__(self): + from infra.config import Settings + self.verbose_flag = Settings.EnableVerboseMessages + Settings.EnableVerboseMessages = False + + def __exit__(self, exc_type, exc_value, traceback): + from infra.config import Settings + Settings.EnableVerboseMessages = self.verbose_flag \ No newline at end of file diff --git a/director/utils/utils.py b/director/utils/utils.py new file mode 100644 index 0000000..fdbc64c --- /dev/null +++ b/director/utils/utils.py @@ -0,0 +1,19 @@ +def str_to_bool(val: str): + if not val: + return False + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return True + elif val in ('', 'n', 'no', 'f', 'false', 'off', '0'): + return False + else: + raise ValueError('invalid truth value %r' % (val,)) + +def trace_origin(initial_frame, stacklevel=0): + frame = initial_frame.f_back + for _ in range(stacklevel + 1): + frame = frame.f_back + file_name = frame.f_code.co_filename + line_number = frame.f_lineno + func_name = frame.f_code.co_name + print(file_name, ":", line_number, ": ", func_name, ": ") diff --git a/director/wrapper.sh b/director/wrapper.sh new file mode 100755 index 0000000..c7beece --- /dev/null +++ b/director/wrapper.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Simple wrapper for executing behave within Docker. +# +# ENVIRONMENT VARIABLES: +# +# - REQUIREMENTS_PATH: requirements fullpath; +# default = "requirements.txt" +# + + +# +# install Python packages for testing purpose, if any. +# + +if [ -z "$REQUIREMENTS_PATH" ]; then + REQUIREMENTS_PATH=requirements.txt +fi + +if [ -f "$REQUIREMENTS_PATH" ]; then + pip3 install --no-cache-dir -r $REQUIREMENTS_PATH +fi + +# +# execute behave +# +exec python -u director.py diff --git a/parser/collect.py b/parser/collect.py index 902165e..e9637e1 100644 --- a/parser/collect.py +++ b/parser/collect.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -from config import Parser +from config import Parser, Mongo token = Parser.GithubToken @@ -111,7 +111,7 @@ def scrape(q, out_file): break #print(all_repos) from pymongo import MongoClient - client = MongoClient("mongodb://admin:admin@localhost:27017") + client = MongoClient(f'mongodb://{Mongo.Username}:{Mongo.Password}@{Mongo.Host}:{Mongo.Port}') db = client['git'] collection = db['repos'] for repo in all_repos: