From eda58ace70f95112e3153ca0dae7fe2970791dd8 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Wed, 19 Mar 2025 21:55:06 +0100 Subject: [PATCH 1/6] feat(CI): add custom docker for CI Refs: #4 Signed-off-by: Klagarge --- .gitlab-ci.yml | 32 +++++++++++++++----------------- Dockerfile | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0412a4..6e5d649 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,33 +1,31 @@ +variables: + DOCKER_IMAGE: registry.forge.hefr.ch/klagarge/mse2425-grp09/python-pdm:latest + default: - image: python:3.9 + image: $DOCKER_IMAGE stages: - lint - - build + - test -build job: - stage: build +test job: + stage: test script: - cd src - # - apt-get update -qy - # - apt-get install -y python3-dev python3-pip python3.12-venv curl - - python3 -V -# - pip3 install --break-system-packages -r requirements.txt - - curl -sSL https://pdm-project.org/install-pdm.py | python3 - - - export PATH=/root/.local/bin:$PATH - - pdm install + - cp -r /app/__pypackages__ . # Set environment variables for the tests - export FLASK_SECRET_KEY=$FLASK_SECRET_KEY # launch tests - - export PYTHONPATH=. + - export PYTHONPATH="/builds/Klagarge/mse2425-grp09/src:/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/lib" + - export PATH="/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/bin:$PATH" - export FLASK_APP=app - pdm run pytest tests --cov --cov-report term --cov-report html artifacts: - paths: - - src/htmlcov/ + paths: + - src/htmlcov/ lint job: stage: lint @@ -38,10 +36,10 @@ lint job: allow_failure: true # Linter can fail, fixing it is for now outside of the projects scope pages: - stage: build + stage: test dependencies: - - build job - needs: ["build job"] + - test job + needs: ["test job"] script: - mv src/htmlcov/ public/ artifacts: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d1e5cd4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.9-slim +LABEL maintener="Rémi Heredero " + +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -sSL https://pdm-project.org/install-pdm.py | python3 - + +ENV PATH="/root/.local/bin:$PATH" \ + PDM_USE_VENV=false + +WORKDIR /app + +COPY src/pyproject.toml src/pdm.lock ./ + +RUN pdm config python.use_venv false && \ + pdm install -G:all + +ENV PATH="/root/.local/bin:$PATH" \ + PDM_USE_VENV=false \ + PYTHONPATH="/app/__pypackages__/3.9/lib" \ + PATH="/app/__pypackages__/3.9/bin:$PATH" \ No newline at end of file From 5e926d76742c7172ac97f9993bd67a14e56c8f68 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Wed, 19 Mar 2025 21:56:44 +0100 Subject: [PATCH 2/6] test(linter): use pdm for linter installation Refs: #5 Signed-off-by: Klagarge --- .gitlab-ci.yml | 7 +++-- src/pdm.lock | 69 +++++++++++++++++++++++++++++++++++++++------- src/pyproject.toml | 10 ++++++- tox.ini | 2 +- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e5d649..9fb31e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,11 @@ lint job: stage: lint dependencies: [] script: - - python3 -m pip install flake8 - - flake8 src/ + - cd src/ + - cp -r /app/__pypackages__ . + - export PYTHONPATH="/builds/Klagarge/mse2425-grp09/src:/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/lib" + - export PATH="/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/bin:$PATH" + - pdm run flake8 --config=../tox.ini allow_failure: true # Linter can fail, fixing it is for now outside of the projects scope pages: diff --git a/src/pdm.lock b/src/pdm.lock index dbc92a9..ea2de1f 100644 --- a/src/pdm.lock +++ b/src/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default"] +groups = ["default", "lint", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:e36fdc748f0c9135da773b2fbab7f45cc5c43e27fad6d39d2de23857da4c1a91" +content_hash = "sha256:f5c4e58e167316cb2440e9205a1e15474a3599463aa47c3c259b4009038166b0" [[metadata.targets]] requires_python = ">3.11" @@ -41,7 +41,7 @@ name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." -groups = ["default"] +groups = ["default", "test"] marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, @@ -53,7 +53,7 @@ name = "coverage" version = "7.6.12" requires_python = ">=3.9" summary = "Code coverage measurement for Python" -groups = ["default"] +groups = ["default", "test"] files = [ {file = "coverage-7.6.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:704c8c8c6ce6569286ae9622e534b4f5b9759b6f2cd643f1c1a61f666d534fe8"}, {file = "coverage-7.6.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ad7525bf0241e5502168ae9c643a2f6c219fa0a283001cee4cf23a9b7da75879"}, @@ -116,7 +116,7 @@ version = "7.6.12" extras = ["toml"] requires_python = ">=3.9" summary = "Code coverage measurement for Python" -groups = ["default"] +groups = ["default", "test"] dependencies = [ "coverage==7.6.12", "tomli; python_full_version <= \"3.11.0a6\"", @@ -189,6 +189,22 @@ files = [ {file = "dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9"}, ] +[[package]] +name = "flake8" +version = "7.1.2" +requires_python = ">=3.8.1" +summary = "the modular source code checker: pep8 pyflakes and co" +groups = ["lint"] +dependencies = [ + "mccabe<0.8.0,>=0.7.0", + "pycodestyle<2.13.0,>=2.12.0", + "pyflakes<3.3.0,>=3.2.0", +] +files = [ + {file = "flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a"}, + {file = "flake8-7.1.2.tar.gz", hash = "sha256:c586ffd0b41540951ae41af572e6790dbd49fc12b3aa2541685d253d9bd504bd"}, +] + [[package]] name = "flask" version = "3.1.0" @@ -229,7 +245,7 @@ name = "iniconfig" version = "2.0.0" requires_python = ">=3.7" summary = "brain-dead simple config-ini parsing" -groups = ["default"] +groups = ["default", "test"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -320,12 +336,23 @@ files = [ {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] +[[package]] +name = "mccabe" +version = "0.7.0" +requires_python = ">=3.6" +summary = "McCabe checker, plugin for flake8" +groups = ["lint"] +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + [[package]] name = "packaging" version = "24.2" requires_python = ">=3.8" summary = "Core utilities for Python packages" -groups = ["default"] +groups = ["default", "test"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -336,18 +363,40 @@ name = "pluggy" version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" -groups = ["default"] +groups = ["default", "test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] +[[package]] +name = "pycodestyle" +version = "2.12.1" +requires_python = ">=3.8" +summary = "Python style guide checker" +groups = ["lint"] +files = [ + {file = "pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3"}, + {file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"}, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +requires_python = ">=3.8" +summary = "passive checker of Python programs" +groups = ["lint"] +files = [ + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, +] + [[package]] name = "pytest" version = "8.3.5" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" -groups = ["default"] +groups = ["default", "test"] dependencies = [ "colorama; sys_platform == \"win32\"", "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", @@ -366,7 +415,7 @@ name = "pytest-cov" version = "6.0.0" requires_python = ">=3.9" summary = "Pytest plugin for measuring coverage." -groups = ["default"] +groups = ["default", "test"] dependencies = [ "coverage[toml]>=7.5", "pytest>=4.6", diff --git a/src/pyproject.toml b/src/pyproject.toml index 2b82610..e373ce2 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -23,4 +23,12 @@ distribution = false [tool.pdm.scripts] flask.cmd = "flask run -p 5000 --debug" flask.env = {FLASK_ENV = "development"} -shell.cmd = "sh" \ No newline at end of file +shell.cmd = "sh" +[dependency-groups] +test = [ + "pytest>=8.3.5", + "pytest-cov>=6.0.0", +] +lint = [ + "flake8>=7.1.2", +] diff --git a/tox.ini b/tox.ini index 8000b14..4082a7e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [flake8] extend-ignore = E203 -exclude = .git,__pycache__,.venv +exclude = .git,__pycache__,.venv,__pypackages__ max-complexity = 10 max-line-length = 120 From 4e27f8c9d91a23c51d44772606b79ebd2c297b79 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Wed, 19 Mar 2025 22:26:38 +0100 Subject: [PATCH 3/6] test(docker): build docker by CI Signed-off-by: Klagarge --- .gitlab-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9fb31e8..15a5db5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ default: image: $DOCKER_IMAGE stages: + - build-docker - lint - test @@ -51,3 +52,21 @@ pages: expire_in: 7 days only: - main + +# This job runs only when Dockerfile changes +docker-build: + image: docker:latest + stage: build-docker + services: + - docker:dind + script: + - docker build -t $DOCKER_IMAGE -f Dockerfile . + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - docker push $DOCKER_IMAGE + rules: + - if: $GITLAB_CI == 'false' # Only run in GitLab CI + when: never + - changes: + - Dockerfile + - src/pyproject.toml + - src/pdm.lock From 8604e3e984256cf5819704a8c2dcb98943947500 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Thu, 20 Mar 2025 13:53:13 +0100 Subject: [PATCH 4/6] docs(CI): answer question 1.4 Refs: #4 Signed-off-by: Klagarge --- docs/questions-part1.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/questions-part1.md b/docs/questions-part1.md index 54264b3..54830f8 100644 --- a/docs/questions-part1.md +++ b/docs/questions-part1.md @@ -14,4 +14,7 @@ ## Q1.3 - A linter is a tool to statically analyse code for readability and improving code quality. It is usually executed from a standalone tool. -- It is used to check for errors, vulns, code smells or general issues but also to enforce a coding style over the whole project. \ No newline at end of file +- It is used to check for errors, vulns, code smells or general issues but also to enforce a coding style over the whole project. + +## Q1.4 +The minimum is to change the image on the CI to put an Alpine or a basic python image. Another option (which we implemented) is to create a custom Docker image that includes all the necessary dependencies for the application. This can significantly reduce the time required to set up the environment and speed up the CI/CD pipeline. From 188a4725e61382bc26c21b358f8fdfb723658ae6 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Thu, 20 Mar 2025 15:45:20 +0100 Subject: [PATCH 5/6] perf(docker): slight optimisation and refector Signed-off-by: Klagarge --- .gitlab-ci.yml | 20 ++++++++++---------- Dockerfile | 11 ++++------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15a5db5..1803ace 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,19 +9,22 @@ stages: - lint - test -test job: - stage: test - script: +.setup_env: &setup_env + before_script: - cd src - cp -r /app/__pypackages__ . + - export "PYTHONPATH=/builds/Klagarge/mse2425-grp09/src:/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/lib" + - export "PATH=/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/bin:$PATH" + - export "FLASK_APP=app" +test job: + stage: test + <<: *setup_env + script: # Set environment variables for the tests - export FLASK_SECRET_KEY=$FLASK_SECRET_KEY # launch tests - - export PYTHONPATH="/builds/Klagarge/mse2425-grp09/src:/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/lib" - - export PATH="/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/bin:$PATH" - - export FLASK_APP=app - pdm run pytest tests --cov --cov-report term --cov-report html artifacts: @@ -30,12 +33,9 @@ test job: lint job: stage: lint + <<: *setup_env dependencies: [] script: - - cd src/ - - cp -r /app/__pypackages__ . - - export PYTHONPATH="/builds/Klagarge/mse2425-grp09/src:/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/lib" - - export PATH="/builds/Klagarge/mse2425-grp09/src/__pypackages__/3.9/bin:$PATH" - pdm run flake8 --config=../tox.ini allow_failure: true # Linter can fail, fixing it is for now outside of the projects scope diff --git a/Dockerfile b/Dockerfile index d1e5cd4..17385ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ -FROM python:3.9-slim +FROM python:3.10-slim LABEL maintener="Rémi Heredero " RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ + pip install --no-cache-dir -U pdm && \ rm -rf /var/lib/apt/lists/* -RUN curl -sSL https://pdm-project.org/install-pdm.py | python3 - ENV PATH="/root/.local/bin:$PATH" \ PDM_USE_VENV=false @@ -17,7 +16,5 @@ COPY src/pyproject.toml src/pdm.lock ./ RUN pdm config python.use_venv false && \ pdm install -G:all -ENV PATH="/root/.local/bin:$PATH" \ - PDM_USE_VENV=false \ - PYTHONPATH="/app/__pypackages__/3.9/lib" \ - PATH="/app/__pypackages__/3.9/bin:$PATH" \ No newline at end of file +ENV PYTHONPATH="/app/__pypackages__/3.9/lib" \ + PATH="/app/__pypackages__/3.9/bin:$PATH" From bcec88f93006b197b12f203320dca888f4653cf9 Mon Sep 17 00:00:00 2001 From: Klagarge Date: Thu, 20 Mar 2025 16:56:01 +0100 Subject: [PATCH 6/6] docs(CI): answer question 2.1 Refs: #6 Signed-off-by: Klagarge --- docs/questions-part2.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/questions-part2.md b/docs/questions-part2.md index 2166737..4f5dbb6 100644 --- a/docs/questions-part2.md +++ b/docs/questions-part2.md @@ -5,3 +5,7 @@ - **Q2.1**: Every commit triggers the CI/CD pipeline. Find out a way to trigger the pipeline only if specific commits (e.g. commit in a development branch) are made. Where can this be configured. Describe your solution and implement it in your pipeline. - **Q2.2**: Take the [CIS controls](./CIS_Controls_v8_Online.22.02.pdf) and give some examples (minimum 5) of controls from this standard that are not or not enough implemented in the calculator app. Provide a short description and a possible remediation. Implement at least two of the controls in the app / pipeline. - **Q2.3 (optional)**: The linter from question 1.3 is a good start. It is only executed in your pipeline. But what if you would also integrate it directly in your local development environment (e.g. IDE)? Can you do the linting before you commit? Describe your solution and implement it in your (local) pipeline. Describe the advantages and disadvantages of this approach. + +# Answers - Part 2 +## Q2.1 +Solution is to add a `rule` section to add condition to trigger the pipeline. It's what is implemented for the `docker-build` job. Another option is to use an `only` section to trigger the pipeline only if the change is made in a specific branch. It's what is implemented for the `pages` job.