Merge patch series "Rework requirements.txt files"

Tom Rini <[email protected]> says:

A challenge we've run in to is making it easier for more people to use
various python tools that we include in the tree. Part of the problem is
that when we have a requirements.txt file, aside from the doc one we
share with the kernel, I created it using "pip freeze". And while this
might have been a best (or at least OK) practice at the time, that's no
longer the case and is why our files have so many things in them. What
this series does is create multiple files, one per project/tool and then
has CI install them as needed. There's a few places here where this
means that we update the requirements as well, but we keep a few big
things where they are currently. This is because updating them
introduces problems of their own and delaing with that would best be a
follow up series. I've put this through GitLab and Azure to make sure
everything is still going fine on both platforms.

Link: https://lore.kernel.org/r/[email protected]

8 files changed, 63 insertions, 54 deletions

diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
index bd197cd6e66..1f2766eae16 100644
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@@ -134,8 +134,11 @@ stages:
           export USER=azure
           virtualenv -p /usr/bin/python3 /tmp/venv
           . /tmp/venv/bin/activate
-          pip install -r test/py/requirements.txt
-          pip install -r tools/buildman/requirements.txt
+          pip install -r test/py/requirements.txt \
+            -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r tools/patman/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt
           export UBOOT_TRAVIS_BUILD_DIR=/tmp/tools-only
           export PYTHONPATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt
           export PATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}
@@ -163,9 +166,14 @@ stages:
       - script: |
           git config --global --add safe.directory $(work_dir)
           export USER=azure
-          pip install -r test/py/requirements.txt
-          pip install -r tools/buildman/requirements.txt
-          pip install asteval pylint==2.12.2 pyopenssl
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r test/py/requirements.txt \
+            -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r tools/patman/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt \
+            asteval pylint==2.12.2 pyopenssl
           export PATH=${PATH}:~/.local/bin
           echo "[MASTER]" >> .pylintrc
           echo "load-plugins=pylint.extensions.docparams" >> .pylintrc
@@ -265,7 +273,13 @@ stages:
           if [ -n "\${BUILD_ENV}" ]; then
               export \${BUILD_ENV};
           fi
-          pip install -r tools/buildman/requirements.txt
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt \
+            -r test/py/requirements.txt \
+            -r tools/u_boot_pylib/requirements.txt \
+            pytest-azurepipelines
           tools/buildman/buildman -o \${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e --board \${TEST_PY_BD} \${OVERRIDE}
           cp /opt/grub/grub_x86.efi \${UBOOT_TRAVIS_BUILD_DIR}/
           cp /opt/grub/grub_x64.efi \${UBOOT_TRAVIS_BUILD_DIR}/
@@ -289,10 +303,6 @@ stages:
               /opt/coreboot/cbfstool \${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom remove -n fallback/payload;
               /opt/coreboot/cbfstool \${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom add-flat-binary -f \${UBOOT_TRAVIS_BUILD_DIR}/u-boot.bin -n fallback/payload -c LZMA -l 0x1110000 -e 0x1110000;
           fi
-          virtualenv -p /usr/bin/python3 /tmp/venv
-          . /tmp/venv/bin/activate
-          pip install -r test/py/requirements.txt
-          pip install pytest-azurepipelines
           export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:\${PATH}
           export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci
           # "\${var:+"-k \$var"}" expands to "" if \$var is empty, "-k \$var" if not
@@ -582,7 +592,10 @@ stages:
           # make environment variables available as tests are running inside a container
           export BUILDMAN="${BUILDMAN}"
           git config --global --add safe.directory ${WORK_DIR}
-          pip install -r tools/buildman/requirements.txt
+          virtualenv -p /usr/bin/python3 /tmp/venv
+          . /tmp/venv/bin/activate
+          pip install -r tools/binman/requirements.txt \
+            -r tools/buildman/requirements.txt
           EOF
           cat << "EOF" >> build.sh
           if [[ "${BUILDMAN}" != "" ]]; then
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 920f61eaa09..e339c25f5e1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,6 +55,11 @@ stages:
         wget -O /tmp/fip.bin https://artifacts.codelinaro.org/artifactory/linaro-419-sbsa-ref/latest/tf-a/fip.bin;
         export BINMAN_INDIRS=/tmp;
       fi
+    # Prepare python environment
+    - virtualenv -p /usr/bin/python3 /tmp/venv;
+      . /tmp/venv/bin/activate;
+      pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/u_boot_pylib/requirements.txt
 
   after_script:
     - cp -v /tmp/${TEST_PY_BD}/*.{html,css,xml} .
@@ -90,9 +95,6 @@ stages:
         /opt/coreboot/cbfstool ${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom remove -n fallback/payload;
         /opt/coreboot/cbfstool ${UBOOT_TRAVIS_BUILD_DIR}/coreboot.rom add-flat-binary -f ${UBOOT_TRAVIS_BUILD_DIR}/u-boot.bin -n fallback/payload -c LZMA -l 0x1110000 -e 0x1110000;
       fi
-    - virtualenv -p /usr/bin/python3 /tmp/venv
-    - . /tmp/venv/bin/activate
-    - pip install -r test/py/requirements.txt
     # "${var:+"-k $var"}" expands to "" if $var is empty, "-k $var" if not
     - export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH};
       export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci;
@@ -120,9 +122,13 @@ build all platforms in a single job:
   tags:
     - ${HOST}
   script:
+    # Prepare python environment
+    - virtualenv -p /usr/bin/python3 /tmp/venv;
+      . /tmp/venv/bin/activate;
+      pip install -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt
     - ret=0;
       git config --global --add safe.directory "${CI_PROJECT_DIR}";
-      pip install -r tools/buildman/requirements.txt;
       ./tools/buildman/buildman -o /tmp -PEWM -x xtensa || ret=$?;
       if [[ $ret -ne 0 ]]; then
         ./tools/buildman/buildman -o /tmp -seP;
@@ -179,8 +185,9 @@ Run binman, buildman, dtoc, Kconfig and patman testsuites:
       export USER=gitlab;
       virtualenv -p /usr/bin/python3 /tmp/venv;
       . /tmp/venv/bin/activate;
-      pip install -r test/py/requirements.txt;
-      pip install -r tools/buildman/requirements.txt;
+      pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/patman/requirements.txt
+        -r tools/u_boot_pylib/requirements.txt;
       export UBOOT_TRAVIS_BUILD_DIR=/tmp/tools-only;
       export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt";
       export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}";
@@ -199,9 +206,11 @@ Run pylint:
   extends: .testsuites
   script:
     - git config --global --add safe.directory "${CI_PROJECT_DIR}"
-    - pip install -r test/py/requirements.txt
-    - pip install -r tools/buildman/requirements.txt
-    - pip install asteval pylint==2.12.2 pyopenssl
+    - virtualenv -p /usr/bin/python3 /tmp/venv
+    - . /tmp/venv/bin/activate
+    - pip install -r test/py/requirements.txt -r tools/binman/requirements.txt
+        -r tools/buildman/requirements.txt -r tools/patman/requirements.txt
+        -r tools/u_boot_pylib/requirements.txt asteval pylint==2.12.2 pyopenssl
     - export PATH=${PATH}:~/.local/bin
     - echo "[MASTER]" >> .pylintrc
     - echo "load-plugins=pylint.extensions.docparams" >> .pylintrc
diff --git a/test/py/requirements.txt b/test/py/requirements.txt
index 75760f96e56..acfe17dce9f 100644
--- a/test/py/requirements.txt
+++ b/test/py/requirements.txt
@@ -1,30 +1,4 @@
-atomicwrites==1.4.1
-attrs==19.3.0
-concurrencytest==0.1.2
-coverage==6.2
-extras==1.0.0
 filelock==3.0.12
-fixtures==3.0.0
-importlib-metadata==0.23
-linecache2==1.0.0
-more-itertools==7.2.0
-packaging==24.1
-pbr==5.4.3
-pluggy==0.13.0
-py==1.11.0
-pycryptodomex==3.19.1
-pyelftools==0.27
-pygit2==1.13.3
-pyparsing==3.0.7
+pycryptodomex==3.21.0
 pytest==6.2.5
 pytest-xdist==2.5.0
-python-mimeparse==1.6.0
-python-subunit==1.3.0
-requests==2.32.3
-setuptools==70.3.0
-six==1.16.0
-testtools==2.3.0
-traceback2==1.4.0
-unittest2==1.1.0
-wcwidth==0.1.7
-zipp==3.19.2
diff --git a/tools/binman/requirements.txt b/tools/binman/requirements.txt
new file mode 100644
index 00000000000..f068ef75a30
--- /dev/null
+++ b/tools/binman/requirements.txt
@@ -0,0 +1,5 @@
+importlib_resources==6.5.2
+jsonschema==4.23.0
+pycryptodomex==3.21.0
+pyelftools==0.31
+yamllint==1.35.1
diff --git a/tools/buildman/requirements.txt b/tools/buildman/requirements.txt
index 052d0ed5c6f..d48650cd1e5 100644
--- a/tools/buildman/requirements.txt
+++ b/tools/buildman/requirements.txt
@@ -1,5 +1,2 @@
-coverage==6.2
-jsonschema==4.17.3
-pycryptodome==3.20
-pyyaml==6.0
-yamllint==1.26.3
+filelock==3.0.12
+importlib_resources==6.5.2
diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile
index d2848ab85f3..85d67848327 100644
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@@ -122,7 +122,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 	python3 \
 	python3-dev \
 	python3-pip \
-	python3-pyelftools \
 	python3-sphinx \
 	python3-virtualenv \
 	rpm2cpio \
@@ -308,12 +307,18 @@ USER uboot:uboot
 # COPY / ADD directives don't work as we need them to.
 RUN wget -O /tmp/pytest-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/test/py/requirements.txt
 RUN wget -O /tmp/sphinx-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/doc/sphinx/requirements.txt
+RUN wget -O /tmp/binman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/binman/requirements.txt
 RUN wget -O /tmp/buildman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/buildman/requirements.txt
+RUN wget -O /tmp/patman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/patman/requirements.txt
+RUN wget -O /tmp/u_boot_pylib-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/u_boot_pylib/requirements.txt
 RUN virtualenv -p /usr/bin/python3 /tmp/venv && \
 	. /tmp/venv/bin/activate && \
 	pip install -r /tmp/pytest-requirements.txt \
 		-r /tmp/sphinx-requirements.txt \
-		-r /tmp/buildman-requirements.txt && \
+		-r /tmp/binman-requirements.txt \
+		-r /tmp/buildman-requirements.txt \
+		-r /tmp/patman-requirements.txt \
+		-r /tmp/u_boot_pylib-requirements.txt && \
 	deactivate && \
 	rm -rf /tmp/venv /tmp/*-requirements.txt
 
diff --git a/tools/patman/requirements.txt b/tools/patman/requirements.txt
new file mode 100644
index 00000000000..e8cbc6cf0c3
--- /dev/null
+++ b/tools/patman/requirements.txt
@@ -0,0 +1,5 @@
+ConfigParser==7.1.0
+importlib_resources==6.5.2
+pygit2==1.13.3
+Requests==2.32.3
+setuptools==75.8.0
diff --git a/tools/u_boot_pylib/requirements.txt b/tools/u_boot_pylib/requirements.txt
new file mode 100644
index 00000000000..1087e6f2857
--- /dev/null
+++ b/tools/u_boot_pylib/requirements.txt
@@ -0,0 +1 @@
+concurrencytest==0.1.2