diff --git a/.gitignore b/.gitignore index 1ce2a87611..bb0faca932 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ cpu.out /bin /dist +/docker/bin /custom/* !/custom/conf /custom/conf/* diff --git a/.woodpecker/testing-amd64.yml b/.woodpecker/compliance-and-tests.yaml similarity index 50% rename from .woodpecker/testing-amd64.yml rename to .woodpecker/compliance-and-tests.yaml index 227c8be268..98e0a239f4 100644 --- a/.woodpecker/testing-amd64.yml +++ b/.woodpecker/compliance-and-tests.yaml @@ -1,13 +1,49 @@ platform: linux/amd64 -depends_on: -- compliance - workspace: base: /go path: src/codeberg/gitea pipeline: + deps-backend: + image: golang:1.19 + pull: true + secrets: + - goproxy_override + commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - make deps-backend + + security-check: + image: golang:1.19 + pull: true + secrets: + - goproxy_override + commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - make security-check + + lint-backend: + image: gitea/test_env:linux-amd64 + pull: true + secrets: + - goproxy_override + environment: + - TAGS=bindata sqlite sqlite_unlock_notify + - GOSUMDB=sum.golang.org + commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - make lint-backend + + checks-backend: + image: golang:1.19 + pull: true + secrets: + - goproxy_override + commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - make --always-make checks-backend + fetch-tags: image: docker:git pull: true @@ -15,12 +51,6 @@ pipeline: - git config --add safe.directory '*' - git fetch --tags --force - deps-backend: - image: golang:1.19 - pull: true - commands: - - make deps-backend - tag-pre-condition: image: drone/git pull: true @@ -30,26 +60,36 @@ pipeline: prepare-test-env: image: gitea/test_env:linux-amd64 pull: true + secrets: + - goproxy_override commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" - ./build/test-env-prepare.sh - build: + build-test: image: gitea/test_env:linux-amd64 + pull: true + secrets: + - goproxy_override environment: - GOSUMDB=sum.golang.org - TAGS=bindata sqlite sqlite_unlock_notify commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" - su gitea -c './build/test-env-check.sh' - su gitea -c 'make backend' unit-test: image: gitea/test_env:linux-amd64 + pull: true + secrets: + - github_read_token + - goproxy_override environment: - TAGS=bindata sqlite sqlite_unlock_notify - RACE_ENABLED=true - secrets: - - github_read_token commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" - su gitea -c 'make unit-test-coverage test-check' test-sqlite: diff --git a/.woodpecker/compliance.yml b/.woodpecker/compliance.yml deleted file mode 100644 index 6077385b8c..0000000000 --- a/.woodpecker/compliance.yml +++ /dev/null @@ -1,32 +0,0 @@ -platform: linux/amd64 - -workspace: - base: /go - path: src/codeberg/gitea - -pipeline: - deps-backend: - image: golang:1.19 - pull: true - commands: - - make deps-backend - - security-check: - image: golang:1.19 - pull: true - commands: - - make security-check - - lint-backend: - image: gitea/test_env:linux-amd64 - pull: true - environment: - - TAGS=bindata sqlite sqlite_unlock_notify - - GOSUMDB=sum.golang.org - commands: - - make lint-backend - - checks-backend: - image: golang:1.19 - commands: - - make --always-make checks-backend diff --git a/.woodpecker/release-version.yml b/.woodpecker/release-version.yml index 2267f3b2bb..b3fcbc961b 100644 --- a/.woodpecker/release-version.yml +++ b/.woodpecker/release-version.yml @@ -1,7 +1,7 @@ platform: linux/amd64 depends_on: -- testing-amd64 +- compliance-and-testing workspace: base: /source @@ -19,24 +19,77 @@ pipeline: image: node:18 pull: true commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" - make deps-frontend + secrets: + - goproxy_override deps-backend: image: golang:1.19 pull: true commands: - - make deps-backend + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - make deps-backend + secrets: + - goproxy_override - static: + build-static: image: techknowlogick/xgo:go-1.19.x pull: true commands: + - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}" + - export WS_BASE="$(pwd)" + - echo "installing nodejs" - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs + - echo "fetching upx" + - cd /usr/local/bin + - curl -sL https://github.com/upx/upx/releases/download/v$${UPXVERSION}/upx-$${UPXVERSION}-amd64_linux.tar.xz | tar --strip-components 1 -xJf - upx-$${UPXVERSION}-amd64_linux/upx + - echo "checking upx" + - test -f /usr/local/bin/upx || exit 1 - export PATH=$PATH:$GOPATH/bin - - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64 release + - echo "building forgejo" + - cd $WS_BASE + - make CI=true LINUX_ARCHS=$${LINUX_ARCHS} release + - echo "building environment-to-ini" + - export TAGS=bindata + - cd $WS_BASE/contrib/environment-to-ini + - make CI=true LINUX_ARCHS=$${LINUX_ARCHS} release environment: TAGS: bindata sqlite sqlite_unlock_notify DEBIAN_FRONTEND: noninteractive + UXPVERSION: 4.0.1 + LINUX_ARCHS: 'linux/amd64,linux/arm64' + secrets: + - goproxy_override + + prep-multiarch-binaries: + image: golang:1.19 # image reuse to reduce pull times, go not required + pull: true + environment: + fallbackversion: main # the fallback version used by make + commands: + - mkdir docker/bin + - ./docker/map-binaries.sh ./dist/binaries forgejo ${CI_COMMIT_TAG##v} + - ./docker/map-binaries.sh ./dist/contrib environment-to-ini ${CI_COMMIT_TAG##v} + when: + event: tag + + build-docker: + image: woodpeckerci/plugin-docker-buildx + pull: true + settings: + dockerfile: Dockerfile.ci + platforms: linux/amd64 + registry: + from_secret: domain + tag: ${CI_COMMIT_TAG##v} + repo: ${CI_REPO_LINK##https://} + password: + from_secret: releaseteamtoken + username: + from_secret: releaseteamuser + when: + event: tag gpg-sign: image: plugins/gpgsign:1 @@ -49,6 +102,8 @@ pipeline: - "dist/release/*" key: from_secret: releaseteamgpg + when: + event: tag release: image: golang:1.19 diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000000..dc7e4bd8bb --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,52 @@ +FROM codeberg.org/forgejo/alpine:3.17.0 +LABEL maintainer="contact@forgejo.org" + +ARG TARGETPLATFORM +ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64} + +EXPOSE 22 3000 + +#regrouped as updates to it should happen less frequently +RUN addgroup \ + -S -g 1000 \ + git && \ + adduser \ + -S -H -D \ + -h /data/git \ + -s /bin/bash \ + -u 1000 \ + -G git \ + git && \ + echo "git:*" | chpasswd -e + +RUN apk --no-cache add \ + bash \ + ca-certificates \ + curl \ + gettext \ + git \ + linux-pam \ + openssh \ + s6 \ + sqlite \ + su-exec \ + gnupg \ + tzdata + +ENV USER git +ENV GITEA_CUSTOM /data/gitea + +VOLUME ["/data"] + +ENTRYPOINT ["/usr/bin/entrypoint"] +CMD ["/bin/s6-svscan", "/etc/s6"] + +COPY docker/root / +COPY docker/bin/$TARGETPLATFORM/forgejo /app/gitea/gitea +COPY docker/bin/$TARGETPLATFORM/environment-to-ini /usr/local/bin/environment-to-ini +# +# s/755/775/ in the following is to avoid the corrupted layer 4f4fb700ef54 +# https://codeberg.org/Codeberg/Community/issues/800#issue-210309 +# +RUN chmod 775 /usr/bin/entrypoint /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini +RUN chmod 775 /etc/s6/gitea/* /etc/s6/openssh/* /etc/s6/.s6-svscan/* diff --git a/contrib/environment-to-ini/Makefile b/contrib/environment-to-ini/Makefile new file mode 100644 index 0000000000..249b7ccda2 --- /dev/null +++ b/contrib/environment-to-ini/Makefile @@ -0,0 +1,146 @@ +ifeq ($(USE_REPO_TEST_DIR),1) + +# This rule replaces the whole Makefile when we're trying to use /tmp repository temporary files +location = $(CURDIR)/$(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) +self := $(location) + +%: + @tmpdir=`mktemp --tmpdir -d` ; \ + echo Using temporary directory $$tmpdir for test repositories ; \ + USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \ + STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS + +else + +# This is the "normal" part of the Makefile + +DIST := ../../dist +DIST_DIRS := $(DIST)/contrib $(DIST)/contrib-release + +GO ?= go +HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" ) +COMMA := , + +XGO_VERSION := go-1.19.x +XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest + +ifeq ($(HAS_GO), GO) + GOPATH ?= $(shell $(GO) env GOPATH) + export PATH := $(GOPATH)/bin:$(PATH) + + CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS) +endif + +EXECUTABLEPREFIX := environment-to-ini + +ifeq ($(shell sed --version 2>/dev/null | grep -q GNU && echo gnu),gnu) + SED_INPLACE := sed -i +else + SED_INPLACE := sed -i '' +endif + +EXTRA_GOFLAGS ?= + +MAKE_VERSION := $(shell "$(MAKE)" -v | head -n 1) + +ifeq ($(RACE_ENABLED),true) + GOFLAGS += -race + GOTESTFLAGS += -race +endif + +STORED_VERSION_FILE := VERSION + +ifneq ($(DRONE_TAG),) + VERSION ?= $(subst v,,$(DRONE_TAG)) + GITEA_VERSION ?= $(VERSION) +else + ifneq ($(DRONE_BRANCH),) + VERSION ?= $(shell echo $(DRONE_BRANCH) | sed -e 's|v\([0-9.][0-9.]*\)/.*|\1|') + else + VERSION ?= main + endif + + STORED_VERSION=$(shell cat $(STORED_VERSION_FILE) 2>/dev/null) + ifneq ($(STORED_VERSION),) + GITEA_VERSION ?= $(STORED_VERSION) + else + GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') + endif +endif + +LDFLAGS := $(LDFLAGS) -s -w -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)" + +LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64 + +GO_PACKAGES ?= $(shell $(GO) list ./... | grep -v /vendor/) + +TAGS ?= +TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS)) + +GO_DIRS := . + +GO_SOURCES := $(wildcard *.go) +GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go") + +.PHONY: go-check +go-check: + $(eval MIN_GO_VERSION_STR := $(shell grep -Eo '^go\s+[0-9]+\.[0-9]+' go.mod | cut -d' ' -f2)) + $(eval MIN_GO_VERSION := $(shell printf "%03d%03d" $(shell echo '$(MIN_GO_VERSION_STR)' | tr '.' ' '))) + $(eval GO_VERSION := $(shell printf "%03d%03d" $(shell $(GO) version | grep -Eo '[0-9]+\.[0-9]+' | tr '.' ' ');)) + @if [ "$(GO_VERSION)" -lt "$(MIN_GO_VERSION)" ]; then \ + echo "Gitea requires Go $(MIN_GO_VERSION_STR) or greater to build. You can get it at https://go.dev/dl/"; \ + exit 1; \ + fi + +.PHONY: clean +clean: + $(GO) clean -i ./... + rm -rf $(DIST) + +.PHONY: generate +generate: $(TAGS_PREREQ) generate-go + +.PHONY: generate-go +generate-go: $(TAGS_PREREQ) + @echo "Running go generate..." + @CC= GOOS= GOARCH= $(GO) generate -tags '$(TAGS)' $(GO_PACKAGES) + +.PHONY: release +release: generate release-linux release-compress release-copy + +$(DIST_DIRS): + mkdir -p $(DIST_DIRS) + +.PHONY: release-windows +release-windows: | $(DIST_DIRS) + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'windows/*' -out $(EXECUTABLEPREFIX)-$(VERSION) . +ifeq ($(CI),true) + cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib +endif + +.PHONY: release-linux +release-linux: | $(DIST_DIRS) + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLEPREFIX)-$(VERSION) . +ifeq ($(CI),true) + cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib +endif + +.PHONY: release-darwin +release-darwin: | $(DIST_DIRS) + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '$(LDFLAGS)' -targets 'darwin-10.12/amd64,darwin-10.12/arm64' -out $(EXECUTABLEPREFIX)-$(VERSION) . +ifeq ($(CI),true) + cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib +endif + +.PHONY: release-compress +release-compress: | $(DIST_DIRS) +ifeq ($(CI),true) + cd $(DIST); for file in `find ./contrib -type f -name "$(EXECUTABLEPREFIX)-*"`; do upx --best --lzma $${file}; done; +endif + +.PHONY: release-copy +release-copy: | $(DIST_DIRS) + cd $(DIST); for file in `find ./contrib -type f -name "$(EXECUTABLEPREFIX)-*"`; do cp $${file} ./contrib-release/; done; + +# This endif closes the if at the top of the file +endif diff --git a/docker/archmap.txt b/docker/archmap.txt new file mode 100644 index 0000000000..f5980a7faa --- /dev/null +++ b/docker/archmap.txt @@ -0,0 +1,9 @@ +# file-name-suffix:docker platform +# quick lookups: https://github.com/techknowlogick/xgo and https://github.com/BretFisher/multi-platform-docker-build#know-what-platforms-you-can-build-in-your-docker-engine +linux-amd64:linux/amd64 +linux-386:linux/386 +linux-arm-6:linux/arm/v6 +linux-arm-7:linux/arm/v7 +linux-arm64:linux/arm64 +linux-riscv64:linux/riscv64 +# add mappings for more platforms below diff --git a/docker/map-binaries.sh b/docker/map-binaries.sh new file mode 100755 index 0000000000..9a30aad540 --- /dev/null +++ b/docker/map-binaries.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +if [[ ${#@} -lt 2 ]]; then + echo "not enough arguments provided" 1>&2 + exit 1 +fi + +BINARYLOOKUPPATH="${1}" +FILEBASENAME="${2}" + +if [[ ${#@} -gt 2 ]] && ! [[ "${3}" == "" ]]; then + FILEVERSION="${3}" +else + FILEVERSION="$FALLBACKVERSION" +fi + +FILE_WIHTOUT_PLATFORM="${FILEBASENAME}-${FILEVERSION}" + +DOCKERDIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +WS_BASE="${DOCKERDIR%/docker}" + +if [[ ! -f "${DOCKERDIR}/archmap.txt" ]]; then + echo "no archmap found" 1>&2 + exit 1 +fi + +function map(){ + local file="${1}" + local bin_platform="${file##./$FILE_WIHTOUT_PLATFORM-}" + if ! grep -q "$bin_platform" "${DOCKERDIR}/archmap.txt"; then + echo "no matching platform for $file found" 2>&1 + return 0 + fi + + local platform="$(grep $bin_platform "${DOCKERDIR}/archmap.txt" | cut -d':' -f2)" + mkdir -p "${DOCKERDIR}/bin/$platform" + echo "Mapped ${file} to $platform, copying" + echo cp "${file}" "${DOCKERDIR}/bin/$platform/${FILEBASENAME}" +} + +cd "$WS_BASE/${BINARYLOOKUPPATH}" + +while IFS= read -r -d $'\0' infile; do + map "$infile" +done < <(find . -maxdepth 1 -type f -name "${FILE_WIHTOUT_PLATFORM}-*" -print0)