[GOBBLIN-2159] Adding support for partition level copy in Iceberg distcp #4121

Workflow file for this run

.github/workflows/build_and_test.yaml at 675e8bb

	#
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	name: Build and Run Tests

	on:
	push:
	# Publish only on `master`
	branches:
	- master
	pull_request:
	branches:
	- master
	release:
	types: [published, edited]

	concurrency:
	group: ci-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	build:
	name: Build repository
	runs-on: ubuntu-latest
	steps:
	- name: Check out the repo
	uses: actions/checkout@v2
	with:
	persist-credentials: false
	- name: Set up JDK 1.8
	uses: actions/setup-java@v1
	with:
	java-version: 1.8
	# Stores external dependencies, can be further improved with Gradle 6.1
	- name: Cache Gradle Dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.gradle/caches
	~/.gradle/wrapper
	# Only rebuild cache if build.gradle is changed
	key: ${{ runner.os }}-gradle-${{ hashFiles('*/.gradle') }}
	restore-keys: ${{ runner.os }}-gradle
	- name: Build repository
	run: \|
	./gradlew --no-daemon clean build -x test -x rat -x javadoc -x findbugsMain -x findbugsTest -x checkstyleMain \
	-x checkstyleJmh -x checkstyleTest -x checkstyleMainGeneratedDataTemplate -x checkstyleMainGeneratedRest -Dorg.gradle.parallel=true
	- name: Verify Dependencies
	run: \|
	# Since dependencies are cached, check after building if they are valid or not
	eval ./gradlew --no-daemon assemble -x javadoc -Dorg.gradle.parallel=true

	static_checks:
	name: Run static checks
	runs-on: ubuntu-latest
	needs: build
	steps:
	- name: Check out the repo
	uses: actions/checkout@v2
	- name: Set up JDK 1.8
	uses: actions/setup-java@v1
	with:
	java-version: 1.8
	# Stores external dependencies, can be further improved with Gradle 6.1
	- name: Cache Gradle Dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.gradle/caches
	~/.gradle/wrapper
	# Only rebuild cache if build.gradle is changed
	key: ${{ runner.os }}-gradle-${{ hashFiles('*/.gradle') }}
	restore-keys: ${{ runner.os }}-gradle
	- name: Run CheckStyle and FindBugs
	run: \|
	./gradlew --no-daemon javadoc findbugsMain checkstyleMain checkstyleTest checkstyleJmh

	run_tests:
	timeout-minutes: 120
	env:
	GOBBLIN_GRADLE_OPTS: "--no-daemon -Dgobblin.metastore.testing.embeddedMysqlEnabled=false -PusePreinstalledMysql=true"
	strategy:
	matrix:
	test-group: ["Core Tests", "Service Tests", "Module Tests", "Other Tests"]
	fail-fast: false
	runs-on: ubuntu-latest
	needs: build
	services:
	mysql:
	image: mysql:8.0.20
	env:
	MYSQL_USER: testUser
	MYSQL_PASSWORD: testPassword
	MYSQL_DATABASE: test
	MYSQL_ROOT_PASSWORD: testPassword
	ports:
	- 3306:3306
	options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
	steps:
	- name: Check out the repo
	uses: actions/checkout@v2
	with:
	persist-credentials: false
	- name: Set up JDK 1.8
	uses: actions/setup-java@v1
	with:
	java-version: 1.8
	# Fix for bug where Github tests are failing port address binding.
	- name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
	run: \|
	echo -e "$(ip addr show eth0 \| grep "inet\b" \| awk '{print $2}' \| cut -d/ -f1)\t$(hostname -f) $(hostname -s)" \| sudo tee -a /etc/hosts
	- name: Verify mysql connection
	run: \|
	sudo apt-get clean
	sudo apt-get --fix-missing update
	sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
	sudo apt-get purge mysql\*
	sudo rm -rf /var/lib/mysql
	sudo rm -rf /etc/mysql
	sudo dpkg -l \| grep -i mysql
	sudo apt-get clean
	sudo apt-get install -y mysql-client
	mysql --host 127.0.0.1 --port 3306 -uroot -ptestPassword -e "SHOW DATABASES"
	mysql --host 127.0.0.1 --port 3306 -uroot -ptestPassword -e "SET GLOBAL max_connections=2000"
	- name: Cache Gradle Dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.gradle/caches
	~/.gradle/wrapper
	# Only rebuild cache if build.gradle is changed
	key: ${{ runner.os }}-gradle-${{ hashFiles('*/.gradle') }}
	restore-keys: ${{ runner.os }}-gradle
	- name: Run test group ${{ matrix.test-group }}
	# Write retry logic as integration tests can fail due to timing out or network problems
	run: \|
	./gradlew getGroupedTests -PgroupName="${{matrix.test-group}}" > temp.txt
	TASKS=$(sed -n 's/CI Task: //p' temp.txt)
	echo $TASKS

	n=0
	until [ "$n" -ge 3 ]
	do
	./gradlew -PskipTestGroup=disabledOnCI $GOBBLIN_GRADLE_OPTS $TASKS -Dorg.gradle.parallel=false && break
	n=$((n+1))
	if [[ $n -lt 3 ]]; then
	echo "Tests failed, retry attempt number $n"
	else
	exit 1
	fi
	sleep 30
	done
	./gradlew -PskipTestGroup=disabledOnCI $GOBBLIN_GRADLE_OPTS -DjacocoBuild=1 jacocoTestReport
	- name: Collect Jacoco Coverage Reports
	run: \|
	JACOCO_REPORTS=$(find . -name "jacoco*.xml" -exec printf ',{}' \; \| cut -c2- )
	echo "jacoco_reports=$JACOCO_REPORTS" >> $GITHUB_ENV
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v2
	with:
	files: ${{ env.jacoco_reports }}
	fail_ci_if_error: false
	verbose: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[GOBBLIN-2159] Adding support for partition level copy in Iceberg distcp #4121

Workflow file

[GOBBLIN-2159] Adding support for partition level copy in Iceberg distcp #4121

Jobs

Run details

Workflow file for this run