diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml index 614fcaec5..5302d696f 100644 --- a/.github/workflows/test-docs.yml +++ b/.github/workflows/test-docs.yml @@ -17,7 +17,7 @@ env: jobs: test-docs: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v5 with: @@ -27,7 +27,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.13" - name: Install requirements run: | diff --git a/.github/workflows/test-ssh.yml b/.github/workflows/test-ssh.yml index 433a92bb5..f465ca577 100644 --- a/.github/workflows/test-ssh.yml +++ b/.github/workflows/test-ssh.yml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: include: - - python: "3.9" + - python: "3.10" - python: "3.12" runs_on: windows-2022 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 083b37e62..cdb0b1f3c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,25 +30,23 @@ jobs: fail-fast: false matrix: include: - - python: "3.9" + - python: "3.11" cluster_type: mpi - - python: "3.10" + - python: "3.12" cluster_type: slurm container: slurmctld - - python: "3.8" - runs_on: 22.04 - python: "3.10" env: IPP_CONTROLLER_IP: "*" - - python: "3.9" + - python: "3.13" env: IPP_ENABLE_CURVE: "1" - - python: "3.8" - runs_on: windows-2019 - - python: "3.9" + - python: "3.10" + runs_on: windows-2025 + - python: "3.12" runs_on: macos-14 - python: "3.11" - - python: "3.12" + - python: "3.14" pre: pre steps: @@ -108,6 +106,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Install ipyparallel itself run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 12ae6fb29..9f67cd10f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,8 +32,8 @@ repos: - --fix # isort rules get confused in the temporary files - --ignore=I - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.6.2 hooks: - id: prettier - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/benchmarks/benchmarks/utils.py b/benchmarks/benchmarks/utils.py index 709d2d73c..b423e107f 100644 --- a/benchmarks/benchmarks/utils.py +++ b/benchmarks/benchmarks/utils.py @@ -1,6 +1,6 @@ import datetime import time -from typing import Callable +from collections.abc import Callable def wait_for(condition: Callable): diff --git a/benchmarks/gcloud_setup.py b/benchmarks/gcloud_setup.py index f73270813..3472728f3 100755 --- a/benchmarks/gcloud_setup.py +++ b/benchmarks/gcloud_setup.py @@ -4,7 +4,6 @@ import sys from subprocess import Popen, check_call from time import sleep -from typing import List import googleapiclient.discovery as gcd @@ -25,7 +24,7 @@ def generate_template_name(number_of_cores_and_ram): return f"{INSTANCE_NAME_PREFIX}{number_of_cores_and_ram}" -def get_running_instance_names() -> List[str]: +def get_running_instance_names() -> list[str]: result = compute.instances().list(project=PROJECT_NAME, zone=ZONE).execute() return [item["name"] for item in result["items"]] if "items" in result else [] diff --git a/ci/slurm/Dockerfile b/ci/slurm/Dockerfile index 039572f2c..3bebe76f1 100644 --- a/ci/slurm/Dockerfile +++ b/ci/slurm/Dockerfile @@ -1,20 +1,57 @@ -# syntax = docker/dockerfile:1.2.1 -FROM ubuntu:20.04 +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt \ rm -f /etc/apt/apt.conf.d/docker-clean \ - && apt-get update && apt-get -y install python3-pip slurm-wlm -ENV PIP_CACHE_DIR=/tmp/pip-cache -RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install ipyparallel pytest-asyncio pytest-cov -RUN mkdir /var/spool/slurmctl \ - && mkdir /var/spool/slurmd -COPY slurm.conf /etc/slurm-llnl/slurm.conf + && apt-get update && apt-get -y install \ + gosu \ + mysql-client \ + python3-venv \ + python3-pip \ + slurm-wlm \ + slurmdbd \ + slurm + +ENV PIP_CACHE_DIR=/tmp/pip-cache \ + VIRTUAL_ENV=/srv/env \ + PATH=/srv/env/bin:${PATH} \ + IPP_DISABLE_JS=1 + +RUN --mount=type=cache,target=${PIP_CACHE_DIR} \ + python3 -m venv $VIRTUAL_ENV \ + && $VIRTUAL_ENV/bin/python3 -m pip install ipyparallel pytest-asyncio pytest-cov + +# initialize some filesystem +RUN mkdir -p /etc/sysconfig/slurm \ + /var/spool/slurmd \ + /var/run/slurmd \ + /var/run/slurmdbd \ + /var/lib/slurmd \ + /data \ + && touch /var/lib/slurmd/node_state \ + /var/lib/slurmd/front_end_state \ + /var/lib/slurmd/job_state \ + /var/lib/slurmd/resv_state \ + /var/lib/slurmd/trigger_state \ + /var/lib/slurmd/assoc_mgr_state \ + /var/lib/slurmd/assoc_usage \ + /var/lib/slurmd/qos_usage \ + /var/lib/slurmd/fed_mgr_state \ + && chown -R slurm:slurm /var/*/slurm* \ + && mkdir /run/munge \ + && chown munge:munge /run/munge \ + && chmod a+rwxt /run/munge + # && mungekey -c + +COPY --chown=slurm:slurm --chmod=0600 etc_slurm/ /etc/slurm/ + + COPY entrypoint.sh /entrypoint -ENV IPP_DISABLE_JS=1 ENTRYPOINT ["/entrypoint"] # the mounted directory RUN mkdir /io ENV PYTHONPATH=/io WORKDIR "/io" + +CMD [ "tail", "-f", "/var/log/slurm/slurmd.log" ] diff --git a/ci/slurm/README.md b/ci/slurm/README.md new file mode 100644 index 000000000..02aeaffda --- /dev/null +++ b/ci/slurm/README.md @@ -0,0 +1,5 @@ +# Slurm cluster example for testing + +adapted and simplified from https://github.com/giovtorres/slurm-docker-cluster/ + +License: MIT diff --git a/ci/slurm/docker-compose.yaml b/ci/slurm/docker-compose.yaml index 87bd5d0a7..54b89a00e 100644 --- a/ci/slurm/docker-compose.yaml +++ b/ci/slurm/docker-compose.yaml @@ -1,84 +1,63 @@ -version: "2.2" - services: - slurmctld: + mysql: + image: mariadb:10.11 + hostname: mysql + container_name: mysql + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: slurm_acct_db + MYSQL_USER: slurm + MYSQL_PASSWORD: password + volumes: + - var_lib_mysql:/var_lib/mysql + + slurmdbd: image: ipp-cluster:slurm build: . - container_name: slurmctld - hostname: slurmctld command: - - tail - - "-f" - - /var/log/slurm-llnl/slurmctld.log + - slurmdbd + container_name: slurmdbd + hostname: slurmdbd volumes: - etc_munge:/etc/munge - - etc_slurm:/etc/slurm - - slurm_jobdir:/data - var_log_slurm:/var/log/slurm - - ../..:/io - expose: - - "6817" - networks: - common-network: - ipv4_address: 10.1.1.10 + depends_on: + - mysql - c1: + slurmctld: image: ipp-cluster:slurm build: . - hostname: c1 command: - - tail - - "-f" - - /var/log/slurm-llnl/slurmd.log - container_name: c1 - + - slurmctld + container_name: slurmctld + hostname: slurmctld volumes: - etc_munge:/etc/munge - - etc_slurm:/etc/slurm - slurm_jobdir:/data - var_log_slurm:/var/log/slurm - ../..:/io - expose: - - "6818" depends_on: - - "slurmctld" - networks: - common-network: - ipv4_address: 10.1.1.11 + - slurmdbd - c2: + compute: image: ipp-cluster:slurm build: . command: - - tail - - "-f" - - /var/log/slurm-llnl/slurmd.log - hostname: c2 - container_name: c2 + - slurmd + deploy: + replicas: 2 + volumes: - etc_munge:/etc/munge - - etc_slurm:/etc/slurm - slurm_jobdir:/data - var_log_slurm:/var/log/slurm - ../..:/io - expose: - - "6818" depends_on: - "slurmctld" - networks: - common-network: - ipv4_address: 10.1.1.12 volumes: etc_munge: - etc_slurm: + # etc_slurm: slurm_jobdir: + var_lib_mysql: var_log_slurm: - -networks: - common-network: - driver: bridge - ipam: - driver: default - config: - - subnet: 10.1.1.0/24 diff --git a/ci/slurm/entrypoint.sh b/ci/slurm/entrypoint.sh index 21cba9e0a..5ec780811 100755 --- a/ci/slurm/entrypoint.sh +++ b/ci/slurm/entrypoint.sh @@ -1,18 +1,64 @@ #!/bin/bash set -ex -# set permissions on munge dir, may be mounted -chown -R munge:munge /etc/munge - -echo "starting munge" -service munge start - -echo "hostname=$(hostname)" -if [[ "$(hostname)" == *"slurmctl"* ]]; then - echo "starting slurmctld" - service slurmctld start -else - echo "starting slurmd" - service slurmd start + +if [ "$1" = "slurmdbd" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Starting the Slurm Database Daemon (slurmdbd) ..." + + { + . /etc/slurm/slurmdbd.conf + until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null + do + echo "-- Waiting for database to become active ..." + sleep 2 + done + } + echo "-- Database is now active ..." + + exec gosu slurm /usr/sbin/slurmdbd -Dvvv +fi + +if [ "$1" = "slurmctld" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Waiting for slurmdbd to become active before starting slurmctld ..." + + until 2>/dev/null >/dev/tcp/slurmdbd/6819 + do + echo "-- slurmdbd is not available. Sleeping ..." + sleep 2 + done + echo "-- slurmdbd is now active ..." + + echo "---> Starting the Slurm Controller Daemon (slurmctld) ..." + if /usr/sbin/slurmctld -V | grep -q '17.02' ; then + exec gosu slurm /usr/sbin/slurmctld -Dvvv + else + exec gosu slurm /usr/sbin/slurmctld -i -Dvvv + fi +fi + +if [ "$1" = "slurmd" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Waiting for slurmctld to become active before starting slurmd..." + + until 2>/dev/null >/dev/tcp/slurmctld/6817 + do + echo "-- slurmctld is not available. Sleeping ..." + sleep 2 + done + echo "-- slurmctld is now active ..." + + echo "---> Starting the Slurm Node Daemon (slurmd) ..." + exec /usr/sbin/slurmd -Z --conf "CPUs=2 RealMemory=1000 Feature=compute" -Dvvv fi exec "$@" diff --git a/ci/slurm/etc_slurm/cgroup.conf b/ci/slurm/etc_slurm/cgroup.conf new file mode 100644 index 000000000..e59e9aeea --- /dev/null +++ b/ci/slurm/etc_slurm/cgroup.conf @@ -0,0 +1 @@ +CgroupPlugin=cgroup/v1 diff --git a/ci/slurm/etc_slurm/slurm.conf b/ci/slurm/etc_slurm/slurm.conf new file mode 100644 index 000000000..d05f997cd --- /dev/null +++ b/ci/slurm/etc_slurm/slurm.conf @@ -0,0 +1,102 @@ +# slurm.conf +# +# See the slurm.conf man page for more information. +# +ClusterName=linux +ControlMachine=slurmctld +ControlAddr=slurmctld +#BackupController= +#BackupAddr= +# +SlurmUser=slurm +#SlurmdUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +#JobCredentialPrivateKey= +#JobCredentialPublicCertificate= +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +#PluginDir= +#CacheGroups=0 +#FirstJobId= +ReturnToService=0 +#MaxJobCount= +#PlugStackConfig= +#PropagatePrioProcess= +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#Prolog= +#Epilog= +#SrunProlog= +#SrunEpilog= +#TaskProlog= +#TaskEpilog= +#TaskPlugin= +#TrackWCKey=no +#TreeWidth=50 +#TmpFS= +#UsePAM= +# +# TIMERS +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +# +# SCHEDULING +SchedulerType=sched/backfill +#SchedulerAuth= +#SchedulerPort= +#SchedulerRootFilter= +SelectType=select/cons_tres # <-- MODIFICAÇÃO 1: Atualizado para compatibilidade com TRES +SelectTypeParameters=CR_Core_Memory +#FastSchedule=1 # <-- MODIFICAÇÃO 2: Comentado por ser obsoleto +#PriorityType=priority/multifactor +#PriorityDecayHalfLife=14-0 +#PriorityUsageResetPeriod=14-0 +#PriorityWeightFairshare=100000 +#PriorityWeightAge=1000 +#PriorityWeightPartition=10000 +#PriorityWeightJobSize=1000 +#PriorityMaxAge=1-0 +# +# LOGGING +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +# +# ACCOUNTING +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +# +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=slurmdbd +AccountingStoragePort=6819 +#AccountingStorageLoc=slurm_acct_db +#AccountingStoragePass= +#AccountingStorageUser= +# +# COMPUTE NODES +#NodeName=c1 NodeAddr=slurm-compute-1 CPUs=2 RealMemory=1000 State=UNKNOWN +#NodeName=c2 NodeAddr=slurm-compute-2 CPUs=2 RealMemory=1000 State=UNKNOWN +# +# PARTITIONS +#PartitionName=normal Default=yes Nodes=c1,c2 Priority=50 DefMemPerCPU=500 Shared=NO MaxNodes=2 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP +# PartitionName=normal Nodes=c1,c2 Default=YES MaxTime=INFINITE State=UP + +# allow dynamic registration of compute nodes (slurmd -Z) +Nodeset=ns1 Feature=compute + +PartitionName=all Nodes=ALL Default=yes MaxNodes=100 +MaxNodeCount=100 diff --git a/ci/slurm/etc_slurm/slurmdbd.conf b/ci/slurm/etc_slurm/slurmdbd.conf new file mode 100644 index 000000000..69d7e1cfc --- /dev/null +++ b/ci/slurm/etc_slurm/slurmdbd.conf @@ -0,0 +1,37 @@ +# +# Example slurmdbd.conf file. +# +# See the slurmdbd.conf man page for more information. +# +# Archive info +#ArchiveJobs=yes +#ArchiveDir="/tmp" +#ArchiveSteps=yes +#ArchiveScript= +#JobPurge=12 +#StepPurge=1 +# +# Authentication info +AuthType=auth/munge +#AuthInfo=/var/run/munge/munge.socket.2 +# +# slurmDBD info +DbdAddr=slurmdbd +DbdHost=slurmdbd +#DbdPort=6819 +SlurmUser=slurm +#MessageTimeout=300 +DebugLevel=4 +#DefaultQOS=normal,standby +LogFile=/var/log/slurm/slurmdbd.log +PidFile=/var/run/slurmdbd/slurmdbd.pid +#PluginDir=/usr/lib/slurm +#PrivateData=accounts,users,usage,jobs +#TrackWCKey=yes +# +# Database info +StorageType=accounting_storage/mysql +StorageHost=mysql +StorageUser=slurm +StoragePass=password +#StorageLoc=slurm_acct_db diff --git a/ci/slurm/slurm.conf b/ci/slurm/slurm.conf deleted file mode 100644 index c75c3fba9..000000000 --- a/ci/slurm/slurm.conf +++ /dev/null @@ -1,59 +0,0 @@ -# slurm.conf file generated by configurator easy.html. -# Put this file on all nodes of your cluster. -# See the slurm.conf man page for more information. -# -SlurmctldHost=slurmctld -# -#MailProg=/bin/mail -MpiDefault=none -#MpiParams=ports=#-# -ProctrackType=proctrack/linuxproc -ReturnToService=1 -SlurmctldPidFile=/var/run/slurmctld.pid -#SlurmctldPort=6817 -SlurmdPidFile=/var/run/slurmd.pid -#SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=root -#SlurmdUser=root -StateSaveLocation=/var/spool/slurmctl -SwitchType=switch/none -#TaskPlugin=task/affinity -#CoreSpecPlugin=core_spec/none - -# -# TIMERS -#KillWait=30 -#MinJobAge=300 -#SlurmctldTimeout=120 -#SlurmdTimeout=300 -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/none -ClusterName=cluster -#JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/none -SlurmctldDebug=debug5 -SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log -SlurmdDebug=debug5 -SlurmdLogFile=/var/log/slurm-llnl/slurmd.log -# -# -# COMPUTE NODES -# Note: CPUs apparently cannot be oversubscribed -# this can only run where at least 2 CPUs are available -#NodeName=localhost NodeHostName=localhost NodeAddr=127.0.0.1 CPUs=2 State=UNKNOWN -#PartitionName=part Nodes=localhost Default=YES MaxTime=INFINITE State=UP OverSubscribe=YES - -# COMPUTE NODES -NodeName=c[1-2] RealMemory=4096 CPUs=2 State=UNKNOWN -# -# PARTITIONS -PartitionName=normal Default=yes Nodes=c[1-2] Priority=50 DefMemPerCPU=2048 Shared=NO MaxNodes=2 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP diff --git a/ci/ssh/Dockerfile b/ci/ssh/Dockerfile index 26869d3ea..953acdc2d 100644 --- a/ci/ssh/Dockerfile +++ b/ci/ssh/Dockerfile @@ -1,5 +1,4 @@ -# syntax = docker/dockerfile:1.2.1 -FROM ubuntu:20.04 +FROM ubuntu:24.04 RUN --mount=type=cache,target=/var/cache/apt \ rm -f /etc/apt/apt.conf.d/docker-clean \ && apt-get update \ @@ -19,7 +18,7 @@ RUN ARCH=$(uname -m | sed s@x86_@@) \ RUN --mount=type=cache,target=${MAMBA_ROOT_PREFIX}/pkgs \ micromamba install -y -p $MAMBA_ROOT_PREFIX -c conda-forge \ - python=3.8 \ + python=3.10 \ pip \ ipyparallel diff --git a/docs/source/conf.py b/docs/source/conf.py index 87293bae0..36c08f7dd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -384,6 +384,10 @@ "https://github.com/search", # github search links "https://github.com/ipython/ipyparallel/compare/", # too many comparisons in changelog r"https?://(localhost|127.0.0.1).*", # ignore localhost references in auto-links + # requests from CI blocked + r"https://www.mcs.anl.gov/.*", + r"https://www.mcs.anl.gov/.*", + r"https://urn.nb.no/.*", ] # myst_nb execution diff --git a/ipyparallel/traitlets.py b/ipyparallel/traitlets.py index 5bd007c0b..09632b7e4 100644 --- a/ipyparallel/traitlets.py +++ b/ipyparallel/traitlets.py @@ -1,11 +1,6 @@ """Custom ipyparallel trait types""" -import sys - -if sys.version_info < (3, 10): - from importlib_metadata import entry_points -else: - from importlib.metadata import entry_points +from importlib.metadata import entry_points from traitlets import List, TraitError, Type diff --git a/pyproject.toml b/pyproject.toml index 2c3255147..658506b8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] urls = {Homepage = "https://ipython.org"} -requires-python = ">=3.8" +requires-python = ">=3.10" dependencies = [ - "importlib_metadata>=3.6; python_version < '3.10'", "decorator", "pyzmq>=25", "traitlets>=5", @@ -106,9 +105,6 @@ artifacts = [ [tool.black] skip-string-normalization = true -target_version = [ - "py38", -] python_cell_magics = ["px"] [tool.isort]