Add pull request checks for Django, Docker, protobuf and React

.github/workflows/django-checks.yml

@@ -0,0 +1,66 @@
+name: "Django checks"
+
+on:
+  pull_request:
+    paths:
+      - "docker/web/**"
+      - "docker/ruff.toml"
+      - ".github/workflows/django-checks.yml"
+  push:
+    branches:
+      - "main"
+      - "devel"
+      - "beta"
+    paths:
+      - "docker/web/**"
+      - "docker/ruff.toml"
+      - ".github/workflows/django-checks.yml"
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # Pinned: ruff's default rules and its findings change between releases.
+      - name: Install ruff
+        run: pip install ruff==0.16.10
+
+      # Only undefined names and syntax errors fail the build for now. The rest of
+      # docker/ruff.toml's rules still report findings (unused imports and the like)
+      # that need a cleanup pass before they can be enforced.
+      - name: Check for undefined names and syntax errors
+        working-directory: docker
+        run: ruff check web/ --select F821,E9 --output-format github
+
+  migrations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12" # Matches the python:3.12 base image in docker/Dockerfile
+          cache: pip
+          cache-dependency-path: docker/web/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r docker/web/requirements.txt
+
+      # settings.py keeps the secret key and database in /data, as in the container.
+      - name: Create the data directory
+        run: sudo install -d -o "$USER" /data
+
+      - name: Run Django system checks
+        working-directory: docker/web/nspanelmanager
+        run: python manage.py check
+
+      # Fails if a model change has no migration, which would leave existing installs
+      # with a database that doesn't match the code after upgrading.
+      - name: Check for missing migrations
+        working-directory: docker/web/nspanelmanager
+        run: python manage.py makemigrations --check --dry-run

.github/workflows/docker-build.yml

@@ -0,0 +1,91 @@
+name: "Docker build"
+
+# Builds the release image without pushing it, so a broken Dockerfile, dependency or cross-compile
+# shows up on the pull request instead of when a release is published. Pull requests and pushes build
+# amd64 and aarch64; the weekly run and manual runs also build the 32-bit platforms.
+
+on:
+  pull_request:
+    paths:
+      - "docker/**"
+      - ".github/workflows/docker-build.yml"
+  push:
+    branches:
+      - "main"
+      - "devel"
+      - "beta"
+    paths:
+      - "docker/**"
+      - ".github/workflows/docker-build.yml"
+  schedule:
+    - cron: "0 4 * * 1"
+  workflow_dispatch:
+
+# A new push to a pull request supersedes its running build, which can take an hour.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  build:
+    name: Build ${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    # Building the Conan dependencies from source takes about an hour when the cache is cold.
+    timeout-minutes: 150
+    strategy:
+      fail-fast: false
+      matrix:
+        # Same architectures as release.yml and docker/config.yaml.
+        arch: ${{ fromJSON((github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["amd64", "aarch64", "armv7", "armhf", "i386"]' || '["amd64", "aarch64"]') }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Map architecture to Docker platform
+        id: platform
+        run: |
+          case "${{ matrix.arch }}" in
+            amd64) echo "platform=linux/amd64" >> "$GITHUB_OUTPUT" ;;
+            aarch64) echo "platform=linux/arm64" >> "$GITHUB_OUTPUT" ;;
+            armv7) echo "platform=linux/arm/v7" >> "$GITHUB_OUTPUT" ;;
+            armhf) echo "platform=linux/arm/v6" >> "$GITHUB_OUTPUT" ;;
+            i386) echo "platform=linux/386" >> "$GITHUB_OUTPUT" ;;
+            *) echo "Unknown architecture ${{ matrix.arch }}"; exit 1 ;;
+          esac
+
+      # MQTTManager is cross-compiled on the runner's platform, but the final stage runs the
+      # target platform's apt-get and pip, which needs emulation.
+      - uses: docker/setup-qemu-action@v3
+
+      - uses: docker/setup-buildx-action@v3
+        id: buildx
+
+      - name: Cache Conan packages
+        id: conan-cache
+        uses: actions/cache@v4
+        with:
+          path: conan-cache
+          key: docker-conan-${{ matrix.arch }}-${{ hashFiles('docker/MQTTManager/conanfile.py', 'docker/build_scripts/install_build_tools.sh') }}
+          restore-keys: docker-conan-${{ matrix.arch }}-
+
+      # Moves the cached directory into the Dockerfile's Conan cache mount before the build,
+      # and back out afterwards so actions/cache can save it.
+      - name: Restore Conan cache mount
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          cache-map: |
+            {
+              "conan-cache": {
+                "target": "/MQTTManager/conan_cache",
+                "id": "nspm-conan",
+                "sharing": "locked"
+              }
+            }
+          skip-extraction: ${{ steps.conan-cache.outputs.cache-hit }}
+
+      - name: Build image
+        uses: docker/build-push-action@v6
+        with:
+          context: docker
+          platforms: ${{ steps.platform.outputs.platform }}
+          push: false

.github/workflows/protobuf.yml

@@ -0,0 +1,60 @@
+name: "Protobuf"
+
+# The code generated from docker/protobuf/*.proto is committed. Check that it matches the .proto
+# files, and that protoc and both protobuf runtimes are on the same release: the generated C++
+# refuses to compile, and the generated Python refuses to import, against a different version.
+
+on:
+  pull_request:
+    paths:
+      - "docker/protobuf/**"
+      - "docker/MQTTManager/include/protobuf/**"
+      - "docker/MQTTManager/conanfile.py"
+      - "docker/web/nspanelmanager/web/protobuf/**"
+      - "docker/web/requirements.txt"
+      - "protobuf_firmware/**"
+      - ".github/workflows/protobuf.yml"
+  push:
+    branches:
+      - "main"
+      - "devel"
+      - "beta"
+    paths:
+      - "docker/protobuf/**"
+      - "docker/MQTTManager/include/protobuf/**"
+      - "docker/MQTTManager/conanfile.py"
+      - "docker/web/nspanelmanager/web/protobuf/**"
+      - "docker/web/requirements.txt"
+      - "protobuf_firmware/**"
+      - ".github/workflows/protobuf.yml"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check protobuf versions match
+        run: |
+          protoc_version=$(sed -nE 's/^REMOTE_FILE="protoc-([0-9.]+)-.*/\1/p' docker/protobuf/install_protobuf.sh)
+          conan_version=$(sed -nE 's/.*"protobuf\/([0-9.]+)".*/\1/p' docker/MQTTManager/conanfile.py)
+          python_version=$(sed -nE 's/^protobuf==([0-9.]+).*/\1/p' docker/web/requirements.txt)
+          echo "protoc $protoc_version, Conan protobuf $conan_version, Python protobuf $python_version"
+          # protoc 27.0 is the protoc of the 5.27.0 C++ and Python runtimes.
+          if [ -z "$protoc_version" ] || [ "5.$protoc_version" != "$conan_version" ] || [ "$conan_version" != "$python_version" ]; then
+            echo "::error::protoc ($protoc_version in install_protobuf.sh), Conan protobuf ($conan_version in conanfile.py) and Python protobuf ($python_version in requirements.txt) must be the same release"
+            exit 1
+          fi
+
+      # The same script contributors run, so a failure here is fixed by running it locally.
+      - name: Regenerate
+        run: bash docker/protobuf/regenerate.sh
+
+      - name: Check generated code is up to date
+        run: |
+          git add --intent-to-add --all
+          if ! git diff --exit-code --stat; then
+            git diff
+            echo "::error::Generated protobuf code is out of date. Run docker/protobuf/regenerate.sh (needs only Docker) and commit the result."
+            exit 1
+          fi

.github/workflows/react-components.yml

@@ -0,0 +1,56 @@
+name: "React components"
+
+# The React components in docker/web/nspanelmanager/web/react-components are built into dist/ and
+# templates/scripts.html, which are committed and served as-is: the Docker image does not rebuild
+# them. Check that they type-check, build, and match what is committed.
+
+on:
+  pull_request:
+    paths:
+      - "docker/web/nspanelmanager/web/react-components/**"
+      - ".github/workflows/react-components.yml"
+  push:
+    branches:
+      - "main"
+      - "devel"
+      - "beta"
+    paths:
+      - "docker/web/nspanelmanager/web/react-components/**"
+      - ".github/workflows/react-components.yml"
+
+defaults:
+  run:
+    working-directory: docker/web/nspanelmanager/web/react-components
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: docker/web/nspanelmanager/web/react-components/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci --no-audit --no-fund
+
+      # "build" runs tsc -b before vite build, so this is also the type check.
+      - name: Type-check and build
+        run: npm run build
+
+      - name: Check the committed build is up to date
+        run: |
+          git add --intent-to-add --all .
+          if ! git diff --exit-code --stat -- .; then
+            echo "::error::dist/ or templates/scripts.html is out of date. Run the build in react-components and commit the result."
+            exit 1
+          fi
+
+      # Reported but not enforced yet: the existing code has 30 errors, mostly no-explicit-any and
+      # react-refresh/only-export-components.
+      - name: Lint
+        continue-on-error: true
+        run: npx eslint .

docker/Dockerfile

@@ -1,12 +1,14 @@
 FROM --platform=$BUILDPLATFORM node:20-alpine AS ui-builder
 ARG BUILDTIME
+# Branch, tag or full commit hash of NSPanelManager-Web-UI to build, e.g. --build-arg WEB_UI_REF=devel.
+ARG WEB_UI_REF=main
 WORKDIR /ui-app
 
 # Install git to clone your frontend repository
 RUN apk add --no-cache git
 
 # Clone the UI repo
-RUN echo "Building UI at $BUILDTIME" && git clone https://github.com/NSPManager/NSPanelManager-Web-UI.git . # Use the echo to force a rebuild of this step.
+RUN echo "Building UI at $BUILDTIME" && git clone https://github.com/NSPManager/NSPanelManager-Web-UI.git . && git fetch origin "$WEB_UI_REF" && git checkout --detach FETCH_HEAD # Use the echo to force a rebuild of this step.
 
 # Install dependencies and build
 RUN npm install
@@ -29,7 +31,11 @@ COPY MQTTManager/ /MQTTManager/
 
 # Only build MQTTManager during Docker build if is is not a devel mode.
 RUN if [ "$IS_DEVEL" != "yes" ]; then bash /build_scripts/install_build_tools.sh "${IS_DEVEL}"; fi
-RUN if [ -z "$no_mqttmanager_build" ]; then /bin/bash /MQTTManager/compile_mqttmanager.sh --target-platform "$TARGETPLATFORM" --strip; else echo "Not building MQTTManager."; fi
+# The Conan package cache (core.cache:storage_path in install_build_tools.sh) is a BuildKit cache
+# mount, so rebuilds reuse the dependencies instead of compiling them from source again, and CI can
+# save it between runs. Only /MQTTManager/build is copied out of this stage.
+RUN --mount=type=cache,id=nspm-conan,target=/MQTTManager/conan_cache,sharing=locked \
+    if [ -z "$no_mqttmanager_build" ]; then /bin/bash /MQTTManager/compile_mqttmanager.sh --target-platform "$TARGETPLATFORM" --strip; else echo "Not building MQTTManager."; fi
 
 FROM python:3.12.5-bookworm
 ARG no_mqttmanager_build

docker/protobuf/compile_all.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
-DEST_DIR_MQTTMANAGER=/MQTTManager/include/protobuf/
-DEST_DIR_DJANGO=/usr/src/app/nspanelmanager/web/protobuf/
-DEST_DIR_FIRMWARE=/full_git/protobuf_firmware/
+# Output directories default to their paths in the dev container; CI overrides them.
+DEST_DIR_MQTTMANAGER=${DEST_DIR_MQTTMANAGER:-/MQTTManager/include/protobuf/}
+DEST_DIR_DJANGO=${DEST_DIR_DJANGO:-/usr/src/app/nspanelmanager/web/protobuf/}
+DEST_DIR_FIRMWARE=${DEST_DIR_FIRMWARE:-/full_git/protobuf_firmware/}
 SRC_FILES=('protobuf_mqttmanager.proto' 'protobuf_general.proto' 'protobuf_formats.proto')
 
 for SRC_FILE in ${SRC_FILES[@]}; do

docker/protobuf/regenerate.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Regenerate the committed C++, Python and firmware C protobuf code from the .proto files in this
+# directory. Runs compile_all.sh in a throwaway container, so the only requirement is Docker. CI's
+# protobuf check runs this same script, so its output is exactly what CI expects.
+#
+#   docker/protobuf/regenerate.sh
+set -euo pipefail
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+
+# Ubuntu 24.04's protobuf-c-compiler is 1.4.1; install_protobuf.sh installs protoc 27.0.
+docker run --rm \
+    -v "$REPO_ROOT":/repo \
+    -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
+    -w /repo/docker/protobuf \
+    ubuntu:24.04 \
+    bash -euo pipefail -c '
+        apt-get update -qq
+        DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \
+            protobuf-c-compiler curl unzip ca-certificates >/dev/null
+        (cd /tmp && bash /repo/docker/protobuf/install_protobuf.sh >/dev/null)
+        protoc --version
+        # -e: compile_all.sh carries on past a failed protoc, which would leave the old files in place.
+        DEST_DIR_MQTTMANAGER=/repo/docker/MQTTManager/include/protobuf/ \
+        DEST_DIR_DJANGO=/repo/docker/web/nspanelmanager/web/protobuf/ \
+        DEST_DIR_FIRMWARE=/repo/protobuf_firmware/ \
+            bash -e compile_all.sh
+        chown -R "$HOST_UID:$HOST_GID" \
+            /repo/docker/MQTTManager/include/protobuf \
+            /repo/docker/web/nspanelmanager/web/protobuf \
+            /repo/protobuf_firmware
+    '

docker/ruff.toml

@@ -1,7 +1,12 @@
 # Allow lines to be as long as 120.
 line-length = 320
+# Migrations and protobuf modules are generated.
+extend-exclude = ["web/nspanelmanager/web/migrations", "web/nspanelmanager/web/protobuf"]
 
 [lint]
+# Ruff's long-standing defaults, listed explicitly so that a newer ruff
+# doesn't enable more rules on its own.
+select = ["E4", "E7", "E9", "F"]
 # Avoid enforcing line-length violations (`E501`)
 ignore = ["E501"]