From 13af3b78eb580b09ea318a0ce1f6ebcf4706f674 Mon Sep 17 00:00:00 2001 From: Evan Date: Mon, 20 Jan 2025 20:09:02 -0500 Subject: [PATCH] Initial commit --- .gitea/workflows/sonarqube.yml | 24 +++++ .gitignore | 177 +++++++++++++++++++++++++++++++++ app/.dockerignore | 4 + app/Dockerfile | 18 ++++ app/__init__.py | 0 app/app.py | 26 +++++ app/config.py | 12 +++ app/docker-compose.yml | 15 +++ app/gunicorn.conf.py | 20 ++++ app/models.py | 5 + app/requirements.txt | 4 + app/routes/__init__.py | 16 +++ app/routes/client.py | 10 ++ app/routes/static.py | 4 + app/utils.py | 49 +++++++++ app/wsgi.py | 10 ++ renovate.json | 7 ++ 17 files changed, 401 insertions(+) create mode 100644 .gitea/workflows/sonarqube.yml create mode 100644 .gitignore create mode 100644 app/.dockerignore create mode 100644 app/Dockerfile create mode 100644 app/__init__.py create mode 100644 app/app.py create mode 100644 app/config.py create mode 100644 app/docker-compose.yml create mode 100644 app/gunicorn.conf.py create mode 100644 app/models.py create mode 100644 app/requirements.txt create mode 100644 app/routes/__init__.py create mode 100644 app/routes/client.py create mode 100644 app/routes/static.py create mode 100644 app/utils.py create mode 100644 app/wsgi.py create mode 100644 renovate.json diff --git a/.gitea/workflows/sonarqube.yml b/.gitea/workflows/sonarqube.yml new file mode 100644 index 0000000..61c3563 --- /dev/null +++ b/.gitea/workflows/sonarqube.yml @@ -0,0 +1,24 @@ +on: + push: + +name: SonarQube Scan +jobs: + sonarqube: + name: SonarQube Trigger + runs-on: ubuntu-latest + steps: + - name: Checking out + uses: actions/checkout@v4 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting + fetch-depth: 0 + - name: get repository name + run: echo "REPOSITORY_NAME=${GITHUB_REPOSITORY#*/}" >> $GITHUB_ENV + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@v4 + env: + SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST }} + with: + args: > + -Dsonar.projectKey=${{ env.REPOSITORY_NAME }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..15c50c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,177 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache +database/ + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# ---> VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + diff --git a/app/.dockerignore b/app/.dockerignore new file mode 100644 index 0000000..39cb4cd --- /dev/null +++ b/app/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +docker-compose.yml +*.tar +__pycache__/ diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 0000000..adcb5b7 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,18 @@ +# Using lightweight alpine image +FROM python:3-alpine + +# Optimize Python behaviors for Docker +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PROMETHEUS_MULTIPROC_DIR=/dev/shm +ENV PAPERSIZE=letter + +# Defining working directory and adding source code +WORKDIR /template +COPY . . + +# Install requirements +RUN python -m pip install --no-cache-dir -r requirements.txt + +# Start app +ENTRYPOINT [ "gunicorn" ] \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/app.py b/app/app.py new file mode 100644 index 0000000..3b7568b --- /dev/null +++ b/app/app.py @@ -0,0 +1,26 @@ +from flask import Flask +import os + +from config import ( + env_SECRET_KEY, + env_SECURE, + env_DEBUG, +) +from utils import logger +from models import db +from routes import routes, metrics + +app = Flask(__name__) +app.secret_key = env_SECRET_KEY +app.config["CORS_HEADERS"] = "Content-Type" +app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///database.db" +db.init_app(app) +app.register_blueprint(routes) +metrics.init_app(app) +app.config["SESSION_COOKIE_SAMESITE"] = "Lax" +app.config["SESSION_COOKIE_SECURE"] = env_SECURE + +logger.info("Worker ready") + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5000, debug=env_DEBUG) diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..1be1c2c --- /dev/null +++ b/app/config.py @@ -0,0 +1,12 @@ +import os +import tempfile + +env_DEBUG = os.environ.get("DEBUG", "").lower() == "true" + +env_SECURE = os.environ.get("SECURE", "").lower() == "true" +env_SECRET_KEY = os.environ.get("SECRET_KEY", os.urandom(24)) +if not env_SECRET_KEY: + env_SECRET_KEY = os.urandom(24) + +path = tempfile.TemporaryDirectory(dir="/dev/shm") +env_TEMP_DIR = path.name diff --git a/app/docker-compose.yml b/app/docker-compose.yml new file mode 100644 index 0000000..54a4c32 --- /dev/null +++ b/app/docker-compose.yml @@ -0,0 +1,15 @@ +services: + template: + container_name: template + image: git.bigun.dev/evan/template:stable + ports: + - 80:5000 # API + - 9200:9200 # Prometheus + restart: unless-stopped + volumes: + - /etc/localtime:/etc/localtime + - ./database:/template/instance + environment: + - DEBUG=FALSE # Enables debug route and Flask's debug mode + - SECRET_KEY="" # Should be a long random value, randomly regenerated every launch if not specified + - SECURE=FALSE # Set to True when using HTTPS diff --git a/app/gunicorn.conf.py b/app/gunicorn.conf.py new file mode 100644 index 0000000..846983a --- /dev/null +++ b/app/gunicorn.conf.py @@ -0,0 +1,20 @@ +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics + +from config import env_DEBUG, env_TEMP_DIR + +accesslog = "-" +access_log_format = '%({X-FORWARDED-FOR}i)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"' +workers = 4 +bind = "0.0.0.0:5000" +wsgi_app = "wsgi:app" +worker_tmp_dir = env_TEMP_DIR +if env_DEBUG: + loglevel = "debug" + + +def when_ready(_): + GunicornPrometheusMetrics.start_http_server_when_ready(9200) + + +def child_exit(_, worker): + GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..9ca8c99 --- /dev/null +++ b/app/models.py @@ -0,0 +1,5 @@ +from datetime import datetime, timezone + +from flask_sqlalchemy import SQLAlchemy + +db = SQLAlchemy() diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..421fbe9 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,4 @@ +Flask==3.1.0 +flask_sqlalchemy==3.1.1 +gunicorn==23.0.0 +prometheus-flask-exporter==0.23.1 \ No newline at end of file diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..456aed2 --- /dev/null +++ b/app/routes/__init__.py @@ -0,0 +1,16 @@ +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics +from flask import Blueprint + +metrics = GunicornPrometheusMetrics() + +by_path_counter = metrics.counter( + "by_path_counter", + "Request count by request paths", + labels={"path": lambda: request.path}, +) + + +routes = Blueprint("routes", __name__) + +from .client import * # Goes last to avoid circular imports +from .static import * diff --git a/app/routes/client.py b/app/routes/client.py new file mode 100644 index 0000000..8646364 --- /dev/null +++ b/app/routes/client.py @@ -0,0 +1,10 @@ +from flask import jsonify, request + +from utils import ( + logger, + validate_data_presence, +) + +# from config import +from . import routes as app +from . import by_path_counter diff --git a/app/routes/static.py b/app/routes/static.py new file mode 100644 index 0000000..e4366b6 --- /dev/null +++ b/app/routes/static.py @@ -0,0 +1,4 @@ +from flask import current_app + +from . import routes as app +from . import by_path_counter diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 0000000..fa073f7 --- /dev/null +++ b/app/utils.py @@ -0,0 +1,49 @@ +from datetime import datetime, timedelta, timezone +import json +from urllib import parse +import logging +import re +import typing as t + +import models + +logger = logging.getLogger("gunicorn.error") + + +def str_none(x): + if x is None: + return "" + else: + return str(x) + + +def string_validator(input_str: str): + # Decode the input string + decoded_str = parse.unquote(input_str) + + # Sanitize the string + sanitized = re.sub(r"[\s]", "", decoded_str) + sanitized = re.sub(r'[<>"\'%;]', "", sanitized) + + # Check length of the string + if len(sanitized) < 1: + return None + + return sanitized + + +def validate_data_presence(data: t.Dict[str, t.Any], keys: list[str]) -> bool: + """ + Validate that all given keys are present in the data. + + Args: + data (Dict[str, Any]): The JSON data to be validated. + keys (list[str]): A list of keys to look for in the data. + + Returns: + bool: If any key is missing, returns False. Otherwise, returns True. + """ + for key in keys: + if key not in data: + return False + return True diff --git a/app/wsgi.py b/app/wsgi.py new file mode 100644 index 0000000..70de510 --- /dev/null +++ b/app/wsgi.py @@ -0,0 +1,10 @@ +from werkzeug.middleware.proxy_fix import ProxyFix +import logging + +from app import app + +gunicorn_logger = logging.getLogger("gunicorn.error") +app.logger.handlers = gunicorn_logger.handlers +app.logger.setLevel(gunicorn_logger.level) + +app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1) diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..be5aad0 --- /dev/null +++ b/renovate.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "local>renovate/renovate-config:default.json" + ] + } + \ No newline at end of file