diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d7ee9615b..d55f212f9 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -29,7 +29,3 @@ jobs: run: ./gradlew :dagger-core:minimalAndShadowPublish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Publish Zip Package of dagger-py-functions - run: ./gradlew publishPyZipPublicationToGitHubPackagesRepository - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/python_release.yml b/.github/workflows/python_release.yml new file mode 100644 index 000000000..4679a3e95 --- /dev/null +++ b/.github/workflows/python_release.yml @@ -0,0 +1,23 @@ +name: Python Package +on: + release: + types: [created] + +jobs: + publishPythonZip: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Zip Python Udf + run: | + zip -r python_udfs.zip dagger-py-functions/udfs -x "*/__init__.py" + zip -r dagger-py-functions.zip python_functions/requirements.txt python_functions/data python_udfs.zip + - name: Upload Release + uses: ncipollo/release-action@v1 + with: + artifacts: dagger-py-functions.zip + allowUpdates: true + omitNameDuringUpdate: true + omitBodyDuringUpdate: true + omitPrereleaseDuringUpdate: true + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/python_validation.yml b/.github/workflows/python_validation.yml new file mode 100644 index 000000000..0a8b04c40 --- /dev/null +++ b/.github/workflows/python_validation.yml @@ -0,0 +1,27 @@ +name: Python Validation + +on: push + +jobs: + pythonValidation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install apache-flink==1.14.3 + cd dagger-py-functions + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Test with pytest + run: | + cd dagger-py-functions + pytest --disable-warnings \ No newline at end of file diff --git a/build.gradle b/build.gradle index 58272b4f1..0af27232b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,25 +1,7 @@ -buildscript { - repositories { - mavenLocal() - mavenCentral() - maven { - url "https://plugins.gradle.org/m2/" - } - } - -} - -plugins { - id 'maven-publish' -} - ext { flinkVersion = System.getenv('flinkVersion') ?: '1.14.3' } -def pyZipVersion = rootProject.file('version.txt').text.trim() -group 'io.odpf' - subprojects { repositories { mavenLocal() @@ -33,37 +15,10 @@ subprojects { apply plugin: 'idea' apply plugin: 'checkstyle' + group 'io.odpf' + checkstyle { toolVersion '7.6.1' configFile rootProject.file("config/checkstyle/checkstyle.xml") } -} - -task makePyZip(type: Zip) { - from fileTree(dir: 'dagger-py-functions') - include '**/*' - archiveName "dagger-py-functions.zip" - destinationDir file("$rootDir/") -} - -publishing { - publications { - pyZip(MavenPublication) { - artifact source: makePyZip, extension: 'zip' - groupId group - artifactId 'dagger-py-functions' - version pyZipVersion - } - } - - repositories { - maven { - name = "GitHubPackages" - url = "https://maven.pkg.github.com/odpf/dagger" - credentials { - username = System.getenv("GITHUB_ACTOR") - password = System.getenv("GITHUB_TOKEN") - } - } - } } \ No newline at end of file diff --git a/dagger-py-functions/data/sample_data.txt b/dagger-py-functions/data/sample_data.txt new file mode 100644 index 000000000..157f4b043 --- /dev/null +++ b/dagger-py-functions/data/sample_data.txt @@ -0,0 +1 @@ +sample_text \ No newline at end of file diff --git a/dagger-py-functions/requirements.txt b/dagger-py-functions/requirements.txt index e69de29bb..64e47665c 100644 --- a/dagger-py-functions/requirements.txt +++ b/dagger-py-functions/requirements.txt @@ -0,0 +1,2 @@ +pytest==7.1.2 +flake8==4.0.1 \ No newline at end of file diff --git a/dagger-py-functions/README.md b/dagger-py-functions/test/__init__.py similarity index 100% rename from dagger-py-functions/README.md rename to dagger-py-functions/test/__init__.py diff --git a/dagger-py-functions/test/scalar/__init__.py b/dagger-py-functions/test/scalar/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dagger-py-functions/test/scalar/sample_test.py b/dagger-py-functions/test/scalar/sample_test.py new file mode 100644 index 000000000..f3f31aec7 --- /dev/null +++ b/dagger-py-functions/test/scalar/sample_test.py @@ -0,0 +1,6 @@ +from udfs.scalar.sample import sample + + +def testSample(): + f = sample._func + assert f("input_text_") == "input_text_sample_text" diff --git a/dagger-py-functions/udfs/scalar/sample.py b/dagger-py-functions/udfs/scalar/sample.py new file mode 100644 index 000000000..6b60f2bd4 --- /dev/null +++ b/dagger-py-functions/udfs/scalar/sample.py @@ -0,0 +1,9 @@ +from pyflink.table import DataTypes +from pyflink.table.udf import udf + + +@udf(result_type=DataTypes.STRING()) +def sample(text): + f = open("data/sample_data.txt", "r") + data = f.read() + return text + data diff --git a/dagger-py-functions/udfs/scalar/sample_udf.py b/dagger-py-functions/udfs/scalar/sample_udf.py deleted file mode 100644 index 786ec1eda..000000000 --- a/dagger-py-functions/udfs/scalar/sample_udf.py +++ /dev/null @@ -1,9 +0,0 @@ -from pyflink.table import ScalarFunction, DataTypes -from pyflink.table.udf import udf - -class SampleUdf(ScalarFunction): - - def eval(self, text: str): - return text + "_added_text" - -sample_udf = udf(SampleUdf(), result_type=DataTypes.STRING()) \ No newline at end of file