From 1bcd8c1649e65e2be82a8839c857355eb4d3196b Mon Sep 17 00:00:00 2001 From: Jay Berkenbilt Date: Fri, 16 Oct 2020 11:17:42 -0400 Subject: [PATCH] Rename azure-pipelines to build-scripts --- README-maintainer | 2 +- azure-pipelines.yml | 14 +++++++------- azure-pipelines/build-appimage | 22 ---------------------- azure-pipelines/build-fuzzer | 11 ----------- azure-pipelines/build-linux | 19 ------------------- azure-pipelines/build-linux32 | 24 ------------------------ azure-pipelines/build-mac | 15 --------------- azure-pipelines/build-windows | 39 --------------------------------------- azure-pipelines/build-windows.bat | 16 ---------------- azure-pipelines/test-sanitizers | 17 ----------------- build-scripts/build-appimage | 22 ++++++++++++++++++++++ build-scripts/build-fuzzer | 11 +++++++++++ build-scripts/build-linux | 19 +++++++++++++++++++ build-scripts/build-linux32 | 24 ++++++++++++++++++++++++ build-scripts/build-mac | 15 +++++++++++++++ build-scripts/build-windows | 39 +++++++++++++++++++++++++++++++++++++++ build-scripts/build-windows.bat | 16 ++++++++++++++++ build-scripts/test-sanitizers | 17 +++++++++++++++++ fuzz/oss-fuzz-build | 2 +- 19 files changed, 172 insertions(+), 172 deletions(-) delete mode 100755 azure-pipelines/build-appimage delete mode 100755 azure-pipelines/build-fuzzer delete mode 100755 azure-pipelines/build-linux delete mode 100755 azure-pipelines/build-linux32 delete mode 100755 azure-pipelines/build-mac delete mode 100755 azure-pipelines/build-windows delete mode 100644 azure-pipelines/build-windows.bat delete mode 100755 azure-pipelines/test-sanitizers create mode 100755 build-scripts/build-appimage create mode 100755 build-scripts/build-fuzzer create mode 100755 build-scripts/build-linux create mode 100755 build-scripts/build-linux32 create mode 100755 build-scripts/build-mac create mode 100755 build-scripts/build-windows create mode 100644 build-scripts/build-windows.bat create mode 100755 build-scripts/test-sanitizers diff --git a/README-maintainer b/README-maintainer index 9f1a4a9..3257719 100644 --- a/README-maintainer +++ b/README-maintainer @@ -326,7 +326,7 @@ When releasing on sourceforge, `external-libs` distributions go in `external-libs/yyyymmdd`, and qpdf distributions go in `qpdf/vvv`. For local iteration on the AppImage generation, it works to just -./azure-pipelines/build-appimage and get the resulting AppImage from +./build-scripts/build-appimage and get the resulting AppImage from the distribution directory. You can also pass -e SKIP_TESTS=1 build-appimage, which passes it along to to docker, to skip the test suite, which useful for rapid iteration. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2010e18..25b499f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,7 +3,7 @@ jobs: pool: vmImage: ubuntu-16.04 steps: - - script: azure-pipelines/build-linux + - script: build-scripts/build-linux displayName: 'Generate, build, and test' - task: PublishBuildArtifacts@1 inputs: @@ -51,7 +51,7 @@ jobs: artifactName: distfiles downloadPath: $(System.DefaultWorkingDirectory) condition: eq(variables['System.PullRequest.IsFork'], 'False') - - script: azure-pipelines/build-windows.bat $(wordsize) $(tool) + - script: build-scripts/build-windows.bat $(wordsize) $(tool) displayName: 'Build, test, generate binary distributions' - task: PublishBuildArtifacts@1 inputs: @@ -75,7 +75,7 @@ jobs: artifactName: distfiles downloadPath: $(System.DefaultWorkingDirectory) condition: eq(variables['System.PullRequest.IsFork'], 'False') - - script: azure-pipelines/build-mac + - script: build-scripts/build-mac displayName: 'Mac build and test' - task: PublishTestResults@2 inputs: @@ -87,7 +87,7 @@ jobs: pool: vmImage: ubuntu-16.04 steps: - - script: azure-pipelines/build-appimage + - script: build-scripts/build-appimage displayName: 'Build AppImage' - task: PublishBuildArtifacts@1 inputs: @@ -105,7 +105,7 @@ jobs: pool: vmImage: ubuntu-16.04 steps: - - script: azure-pipelines/build-linux32 + - script: build-scripts/build-linux32 displayName: 'Linux 32-bit' - task: PublishTestResults@2 inputs: @@ -117,7 +117,7 @@ jobs: pool: vmImage: ubuntu-16.04 steps: - - script: azure-pipelines/build-fuzzer + - script: build-scripts/build-fuzzer displayName: 'Build Fuzzer' dependsOn: Linux condition: succeeded() @@ -125,7 +125,7 @@ jobs: pool: vmImage: ubuntu-16.04 steps: - - script: azure-pipelines/test-sanitizers + - script: build-scripts/test-sanitizers displayName: 'Sanitizer Tests' - task: PublishTestResults@2 inputs: diff --git a/azure-pipelines/build-appimage b/azure-pipelines/build-appimage deleted file mode 100755 index 8a61470..0000000 --- a/azure-pipelines/build-appimage +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Any extra args are passed to the docker run command before the -# invocation of qpdfbuild. This is useful for iterating locally as -# described in README-maintainer. -# -set -ex -cd appimage -docker build -t qpdfbuild . -rm -rf build -mkdir build -cd .. -git clone .git appimage/build/qpdf -docker run --privileged --rm \ - -v $PWD/appimage/build:/tmp/build ${1+"$@"} qpdfbuild -rm -rf distribution -mkdir distribution -cp -p appimage/build/qpdf/appimage/build/qpdf*AppImage* distribution -for i in distribution/*; do - mv $i $(echo $i | sed -e 's/\.AppImage/-ci.AppImage/') -done -sha256sum distribution/* diff --git a/azure-pipelines/build-fuzzer b/azure-pipelines/build-fuzzer deleted file mode 100755 index 00c1c8a..0000000 --- a/azure-pipelines/build-fuzzer +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -ex -export WORK=$PWD/work -export OUT=$PWD/out -mkdir -p $WORK $OUT -sudo apt-get update -sudo apt-get -y install \ - autoconf build-essential zlib1g-dev libjpeg-dev -./fuzz/oss-fuzz-build -ls -l out/qpdf*fuzzer -ls -l out/ diff --git a/azure-pipelines/build-linux b/azure-pipelines/build-linux deleted file mode 100755 index 266b992..0000000 --- a/azure-pipelines/build-linux +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -ex -sudo apt-get update -sudo apt-get -y install \ - autoconf build-essential zlib1g-dev libjpeg-dev libgnutls28-dev \ - libssl-dev docbook-xsl fop xsltproc libxml2-utils inkscape imagemagick -./configure --enable-werror --enable-doc-maintenance \ - --enable-crypto-native --enable-crypto-openssl --enable-crypto-gnutls \ - --enable-show-failed-test-output -make -j$(nproc) -k -for i in $(./qpdf/build/qpdf --show-crypto); do - echo "*** Running tests with crypto provider $i" - env QPDF_CRYPTO_PROVIDER=$i make -k check -done -make distfiles.zip -./make_dist --ci --no-tests -mkdir distribution -cp /tmp/qpdf*-ci.tar.gz distribution -sha256sum distribution/* diff --git a/azure-pipelines/build-linux32 b/azure-pipelines/build-linux32 deleted file mode 100755 index 2c51627..0000000 --- a/azure-pipelines/build-linux32 +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -ex - -# dpkg --add-architecture not needed for Ubuntu 18.04 -sudo dpkg --add-architecture i386 - -sudo apt-get update - -# For Ubuntu 18.04: -# -# sudo apt-get -y install \ -# autoconf build-essential zlib1g-dev:i386 libjpeg-dev:i386 \ -# g++-multilib-i686-linux-gnu -# ./configure --enable-werror --enable-show-failed-test-output \ -# CC=i686-linux-gnu-gcc CXX=i686-linux-gnu-g++ - -sudo apt-get -y install \ - autoconf build-essential zlib1g-dev:i386 libjpeg-dev:i386 \ - libssl-dev:i386 g++-multilib -./configure --enable-werror --enable-show-failed-test-output \ - CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 - -make -j$(nproc) -k -make -k check diff --git a/azure-pipelines/build-mac b/azure-pipelines/build-mac deleted file mode 100755 index 96d900b..0000000 --- a/azure-pipelines/build-mac +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -ex -curl -L https://github.com/qpdf/qpdf/raw/external-libs/jpegsrc.v9c.tar.gz -o jpegsrc.v9c.tar.gz -tar xzf jpegsrc.v9c.tar.gz -cd jpeg-9c -./configure -make -k -sudo make install -cd .. -if [ -f distfiles/distfiles.zip ]; then - unzip distfiles/distfiles.zip -fi -./configure --enable-werror --enable-show-failed-test-output -make -j$(nproc) -k -make -k check diff --git a/azure-pipelines/build-windows b/azure-pipelines/build-windows deleted file mode 100755 index 9e902e5..0000000 --- a/azure-pipelines/build-windows +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -ex -cd $(dirname $0)/.. -wordsize=$1 -tool=$2 - -if [[ $tool == mingw ]]; then - pacman -Sy --noconfirm make base-devel tar zip unzip - if [[ $wordsize == 64 ]]; then - pacman -Sy --noconfirm mingw-w64-x86_64-toolchain - PATH="/c/tools/msys64/mingw64/bin:$PATH" - else - pacman -Sy --noconfirm mingw-w64-i686-toolchain - PATH="/c/tools/msys64/mingw32/bin:$PATH" - fi - g++ -v -elif [[ $tool == msvc ]]; then - cl -fi -if [ -f distfiles/distfiles.zip ]; then - unzip distfiles/distfiles.zip -fi -curl -L https://github.com/qpdf/qpdf/raw/external-libs/qpdf-external-libs-bin.zip -o qpdf-external-libs-bin.zip -unzip qpdf-external-libs-bin.zip -cwd=`pwd` -PATH=$cwd/libqpdf/build:$PATH - -installdir=install-$tool$wordsize -rm -rf $installdir -./config-$tool --enable-show-failed-test-output --disable-crypto-gnutls --disable-crypto-openssl -make -j$(nproc) -k -make -k check -make install - -v=`(cd $installdir; ls -d qpdf-*)` -cp -p README-windows-install.txt $installdir/$v/README.txt -mkdir distribution -(cd $installdir; zip -r ../distribution/$v-bin-$tool$wordsize-ci.zip $v) -sha256sum distribution/* diff --git a/azure-pipelines/build-windows.bat b/azure-pipelines/build-windows.bat deleted file mode 100644 index e646f0a..0000000 --- a/azure-pipelines/build-windows.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo on -@rem Usage: build-windows {32|64} {msvc|mingw} -if %2 == msvc ( - if %1 == 64 ( - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ) else ( - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars32.bat" - ) - choco install zip - bash ./azure-pipelines/build-windows %1 %2 -) else ( - @rem The vs2017-win2016 pool has an ancient 64-bit-only mingw. - @rem Install msys2 so we can get current gcc toolchains. - choco install msys2 - C:\tools\msys64\usr\bin\env.exe MSYSTEM=MINGW64 /bin/bash -l %CD%/azure-pipelines/build-windows %1 %2 -) diff --git a/azure-pipelines/test-sanitizers b/azure-pipelines/test-sanitizers deleted file mode 100755 index 476f198..0000000 --- a/azure-pipelines/test-sanitizers +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -sudo apt-get update -sudo apt-get -y install \ - autoconf build-essential zlib1g-dev libjpeg-dev libgnutls28-dev libssl-dev -./configure \ - CFLAGS="-fsanitize=address -fsanitize=undefined -g" \ - CXXFLAGS="-fsanitize=address -fsanitize=undefined -g" \ - LDFLAGS="-fsanitize=address -fsanitize=undefined" \ - CC=clang CXX="clang++ --std=c++11" \ - --enable-crypto-native --enable-crypto-openssl --enable-crypto-gnutls \ - --enable-werror --disable-shared --enable-show-failed-test-output -make -j$(nproc) -k -for i in $(./qpdf/build/qpdf --show-crypto); do - echo "*** Running tests with crypto provider $i" - env QPDF_CRYPTO_PROVIDER=$i make -k check -done diff --git a/build-scripts/build-appimage b/build-scripts/build-appimage new file mode 100755 index 0000000..8a61470 --- /dev/null +++ b/build-scripts/build-appimage @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Any extra args are passed to the docker run command before the +# invocation of qpdfbuild. This is useful for iterating locally as +# described in README-maintainer. +# +set -ex +cd appimage +docker build -t qpdfbuild . +rm -rf build +mkdir build +cd .. +git clone .git appimage/build/qpdf +docker run --privileged --rm \ + -v $PWD/appimage/build:/tmp/build ${1+"$@"} qpdfbuild +rm -rf distribution +mkdir distribution +cp -p appimage/build/qpdf/appimage/build/qpdf*AppImage* distribution +for i in distribution/*; do + mv $i $(echo $i | sed -e 's/\.AppImage/-ci.AppImage/') +done +sha256sum distribution/* diff --git a/build-scripts/build-fuzzer b/build-scripts/build-fuzzer new file mode 100755 index 0000000..00c1c8a --- /dev/null +++ b/build-scripts/build-fuzzer @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex +export WORK=$PWD/work +export OUT=$PWD/out +mkdir -p $WORK $OUT +sudo apt-get update +sudo apt-get -y install \ + autoconf build-essential zlib1g-dev libjpeg-dev +./fuzz/oss-fuzz-build +ls -l out/qpdf*fuzzer +ls -l out/ diff --git a/build-scripts/build-linux b/build-scripts/build-linux new file mode 100755 index 0000000..266b992 --- /dev/null +++ b/build-scripts/build-linux @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex +sudo apt-get update +sudo apt-get -y install \ + autoconf build-essential zlib1g-dev libjpeg-dev libgnutls28-dev \ + libssl-dev docbook-xsl fop xsltproc libxml2-utils inkscape imagemagick +./configure --enable-werror --enable-doc-maintenance \ + --enable-crypto-native --enable-crypto-openssl --enable-crypto-gnutls \ + --enable-show-failed-test-output +make -j$(nproc) -k +for i in $(./qpdf/build/qpdf --show-crypto); do + echo "*** Running tests with crypto provider $i" + env QPDF_CRYPTO_PROVIDER=$i make -k check +done +make distfiles.zip +./make_dist --ci --no-tests +mkdir distribution +cp /tmp/qpdf*-ci.tar.gz distribution +sha256sum distribution/* diff --git a/build-scripts/build-linux32 b/build-scripts/build-linux32 new file mode 100755 index 0000000..2c51627 --- /dev/null +++ b/build-scripts/build-linux32 @@ -0,0 +1,24 @@ +#!/bin/bash +set -ex + +# dpkg --add-architecture not needed for Ubuntu 18.04 +sudo dpkg --add-architecture i386 + +sudo apt-get update + +# For Ubuntu 18.04: +# +# sudo apt-get -y install \ +# autoconf build-essential zlib1g-dev:i386 libjpeg-dev:i386 \ +# g++-multilib-i686-linux-gnu +# ./configure --enable-werror --enable-show-failed-test-output \ +# CC=i686-linux-gnu-gcc CXX=i686-linux-gnu-g++ + +sudo apt-get -y install \ + autoconf build-essential zlib1g-dev:i386 libjpeg-dev:i386 \ + libssl-dev:i386 g++-multilib +./configure --enable-werror --enable-show-failed-test-output \ + CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 + +make -j$(nproc) -k +make -k check diff --git a/build-scripts/build-mac b/build-scripts/build-mac new file mode 100755 index 0000000..96d900b --- /dev/null +++ b/build-scripts/build-mac @@ -0,0 +1,15 @@ +#!/bin/bash +set -ex +curl -L https://github.com/qpdf/qpdf/raw/external-libs/jpegsrc.v9c.tar.gz -o jpegsrc.v9c.tar.gz +tar xzf jpegsrc.v9c.tar.gz +cd jpeg-9c +./configure +make -k +sudo make install +cd .. +if [ -f distfiles/distfiles.zip ]; then + unzip distfiles/distfiles.zip +fi +./configure --enable-werror --enable-show-failed-test-output +make -j$(nproc) -k +make -k check diff --git a/build-scripts/build-windows b/build-scripts/build-windows new file mode 100755 index 0000000..9e902e5 --- /dev/null +++ b/build-scripts/build-windows @@ -0,0 +1,39 @@ +#!/bin/bash +set -ex +cd $(dirname $0)/.. +wordsize=$1 +tool=$2 + +if [[ $tool == mingw ]]; then + pacman -Sy --noconfirm make base-devel tar zip unzip + if [[ $wordsize == 64 ]]; then + pacman -Sy --noconfirm mingw-w64-x86_64-toolchain + PATH="/c/tools/msys64/mingw64/bin:$PATH" + else + pacman -Sy --noconfirm mingw-w64-i686-toolchain + PATH="/c/tools/msys64/mingw32/bin:$PATH" + fi + g++ -v +elif [[ $tool == msvc ]]; then + cl +fi +if [ -f distfiles/distfiles.zip ]; then + unzip distfiles/distfiles.zip +fi +curl -L https://github.com/qpdf/qpdf/raw/external-libs/qpdf-external-libs-bin.zip -o qpdf-external-libs-bin.zip +unzip qpdf-external-libs-bin.zip +cwd=`pwd` +PATH=$cwd/libqpdf/build:$PATH + +installdir=install-$tool$wordsize +rm -rf $installdir +./config-$tool --enable-show-failed-test-output --disable-crypto-gnutls --disable-crypto-openssl +make -j$(nproc) -k +make -k check +make install + +v=`(cd $installdir; ls -d qpdf-*)` +cp -p README-windows-install.txt $installdir/$v/README.txt +mkdir distribution +(cd $installdir; zip -r ../distribution/$v-bin-$tool$wordsize-ci.zip $v) +sha256sum distribution/* diff --git a/build-scripts/build-windows.bat b/build-scripts/build-windows.bat new file mode 100644 index 0000000..8be269e --- /dev/null +++ b/build-scripts/build-windows.bat @@ -0,0 +1,16 @@ +@echo on +@rem Usage: build-windows {32|64} {msvc|mingw} +if %2 == msvc ( + if %1 == 64 ( + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + ) else ( + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars32.bat" + ) + choco install zip + bash ./build-scripts/build-windows %1 %2 +) else ( + @rem The vs2017-win2016 pool has an ancient 64-bit-only mingw. + @rem Install msys2 so we can get current gcc toolchains. + choco install msys2 + C:\tools\msys64\usr\bin\env.exe MSYSTEM=MINGW64 /bin/bash -l %CD%/build-scripts/build-windows %1 %2 +) diff --git a/build-scripts/test-sanitizers b/build-scripts/test-sanitizers new file mode 100755 index 0000000..476f198 --- /dev/null +++ b/build-scripts/test-sanitizers @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +sudo apt-get update +sudo apt-get -y install \ + autoconf build-essential zlib1g-dev libjpeg-dev libgnutls28-dev libssl-dev +./configure \ + CFLAGS="-fsanitize=address -fsanitize=undefined -g" \ + CXXFLAGS="-fsanitize=address -fsanitize=undefined -g" \ + LDFLAGS="-fsanitize=address -fsanitize=undefined" \ + CC=clang CXX="clang++ --std=c++11" \ + --enable-crypto-native --enable-crypto-openssl --enable-crypto-gnutls \ + --enable-werror --disable-shared --enable-show-failed-test-output +make -j$(nproc) -k +for i in $(./qpdf/build/qpdf --show-crypto); do + echo "*** Running tests with crypto provider $i" + env QPDF_CRYPTO_PROVIDER=$i make -k check +done diff --git a/fuzz/oss-fuzz-build b/fuzz/oss-fuzz-build index 0a05657..a875edc 100755 --- a/fuzz/oss-fuzz-build +++ b/fuzz/oss-fuzz-build @@ -4,7 +4,7 @@ # https://github.com/google/oss-fuzz/blob/master/projects/qpdf/build.sh # It should be run from the top level directory of a clean checkout of -# qpdf. It is also exercised in ../azure-pipelines/build-fuzzer +# qpdf. It is also exercised in ../build-scripts/build-fuzzer if [[ $GITHUB_FORK != "" ]]; then git remote add fork https://github.com/$GITHUB_FORK/qpdf -- libgit2 0.21.4