From b5799a5a4af75e53c219bb8eb1cea346917f1841 Mon Sep 17 00:00:00 2001 From: Ulada Zakharava Date: Fri, 24 Apr 2026 14:40:17 +0000 Subject: [PATCH] Add aliases for rebranded Google services --- .../google/docs/connections/gcp_looker.rst | 32 +-- .../integration-logos/Google-Data-Proc.png | Bin 24546 -> 0 bytes .../google/docs/operators/cloud/dataplex.rst | 204 ++++++++++++++---- .../google/docs/operators/cloud/dataproc.rst | 120 ++++++++--- .../google/docs/operators/cloud/looker.rst | 24 ++- providers/google/provider.yaml | 36 ++-- .../cloud/operators/knowledge_catalog.py | 112 ++++++++++ .../google/cloud/operators/looker.py | 7 +- .../google/cloud/operators/managed_spark.py | 56 +++++ .../providers/google/get_provider_info.py | 46 ++-- .../google/cloud/dataplex/example_dataplex.py | 4 +- .../dataplex/example_dataplex_catalog.py | 2 +- .../cloud/dataplex/example_dataplex_dp.py | 4 +- .../cloud/dataplex/example_dataplex_dq.py | 4 +- .../cloud/dataproc/example_dataproc_batch.py | 4 +- .../example_dataproc_batch_deferrable.py | 2 +- .../example_dataproc_batch_persistent.py | 4 +- ...cluster_create_existing_stopped_cluster.py | 6 +- .../example_dataproc_cluster_deferrable.py | 2 +- .../example_dataproc_cluster_diagnose.py | 2 +- .../example_dataproc_cluster_generator.py | 4 +- .../example_dataproc_cluster_start_stop.py | 6 +- .../example_dataproc_cluster_update.py | 2 +- .../cloud/dataproc/example_dataproc_flink.py | 2 +- .../cloud/dataproc/example_dataproc_gke.py | 4 +- .../cloud/dataproc/example_dataproc_hadoop.py | 2 +- .../cloud/dataproc/example_dataproc_hive.py | 2 +- .../cloud/dataproc/example_dataproc_pig.py | 2 +- .../cloud/dataproc/example_dataproc_presto.py | 2 +- .../dataproc/example_dataproc_pyspark.py | 2 +- .../cloud/dataproc/example_dataproc_spark.py | 2 +- .../dataproc/example_dataproc_spark_async.py | 2 +- .../example_dataproc_spark_deferrable.py | 2 +- .../dataproc/example_dataproc_spark_sql.py | 2 +- .../cloud/dataproc/example_dataproc_sparkr.py | 2 +- .../example_dataproc_start_from_trigger.py | 2 +- .../cloud/dataproc/example_dataproc_trino.py | 2 +- .../dataproc/example_dataproc_workflow.py | 4 +- .../example_dataproc_workflow_deferrable.py | 4 +- .../google/cloud/looker/example_looker.py | 4 +- .../google/cloud/operators/test_dataproc.py | 1 + .../cloud/operators/test_knowledge_catalog.py | 171 +++++++++++++++ .../google/cloud/operators/test_looker.py | 9 +- .../cloud/operators/test_managed_spark.py | 37 ++++ 44 files changed, 760 insertions(+), 183 deletions(-) delete mode 100644 providers/google/docs/integration-logos/Google-Data-Proc.png create mode 100644 providers/google/src/airflow/providers/google/cloud/operators/knowledge_catalog.py create mode 100644 providers/google/src/airflow/providers/google/cloud/operators/managed_spark.py create mode 100644 providers/google/tests/unit/google/cloud/operators/test_knowledge_catalog.py create mode 100644 providers/google/tests/unit/google/cloud/operators/test_managed_spark.py diff --git a/providers/google/docs/connections/gcp_looker.rst b/providers/google/docs/connections/gcp_looker.rst index 8f3099c8a2363..1b7dc344c1a55 100644 --- a/providers/google/docs/connections/gcp_looker.rst +++ b/providers/google/docs/connections/gcp_looker.rst @@ -17,42 +17,42 @@ .. _howto/connection:gcp_looker: -Google Cloud Platform Looker Connection -======================================= +Google Cloud Platform Data Studio (Looker) Connection +====================================================== -Communication between Airflow and Looker is done via `Looker API `_. -To facilitate the API communication Looker operators use `Looker SDK `_ as an API client. -Before calling API, Looker SDK needs to authenticate itself using your Looker API credentials. +Communication between Airflow and Data Studio (Looker) is done via `Looker API `_. +To facilitate the API communication, Data Studio operators use `Looker SDK `_ as an API client. +Before calling API, Looker SDK needs to authenticate itself using your Data Studio API credentials. -* Obtain your Looker API credentials using instructions in the `Looker API authentication documentation `_. +* Obtain your Data Studio API credentials using instructions in the `Looker API authentication documentation `_. -* Obtain your Looker API path and port as described in the `Looker API documentation `_. +* Obtain your Data Studio API path and port as described in the `Looker API documentation `_. -* Setup a Looker connection in Airflow. +* Setup a Data Studio connection in Airflow. -The ``HTTP`` connection type provides connection to Looker API. +The ``HTTP`` connection type provides connection to Data Studio API. The :class:`~airflow.providers.google.cloud.hooks.looker.LookerHook` uses this connection to run -API requests on a Looker instance issued by :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` and :class:`~airflow.providers.google.cloud.sensors.looker.LookerCheckPdtBuildSensor`. +API requests on a Data Studio instance issued by :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` and :class:`~airflow.providers.google.cloud.sensors.looker.LookerCheckPdtBuildSensor`. Configuring the Connection -------------------------- Host (required) - Base URL for Looker API. Do not include /api/* in the URL. + Base URL for Data Studio API. Do not include /api/* in the URL. Login (required) - Looker API client id. + Data Studio API client id. Password (required) - Looker API client secret. + Data Studio API client secret. Port (optional) - Port for Looker API. If hosted on GCP, don't specify the port leaving just the host. + Port for Data Studio API. If hosted on GCP, don't specify the port leaving just the host. Extra (optional) - Specify the extra parameters (as json dictionary) that can be used in Looker + Specify the extra parameters (as json dictionary) that can be used in Data Studio connection. The following parameters are supported: * ``verify_ssl`` - Set to false only if testing locally against self-signed certs. Defaults to true if not specified. @@ -70,7 +70,7 @@ Extra (optional) Connection URI -------------- -A URL configuration example of a Looker connection: +A URL configuration example of a Data Studio connection: .. code-block:: diff --git a/providers/google/docs/integration-logos/Google-Data-Proc.png b/providers/google/docs/integration-logos/Google-Data-Proc.png deleted file mode 100644 index 914c384f21ec79c96be9df4496bc6ea4c88dd42e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24546 zcma%hV{oM36Yj*u#I|kQcCr(0%ndfS?PO!ywvFA`$;KPo=FRVa>wdpg^VUqAcTS&n zKYgkrRFtHV;PK!A005G#jD#8h0LJj&0}K7N;^7nP0sv6Rduo83)r{Oo931V+5gvH(!SI2yYKuep|@CfB!(f@yf>$s$F~Bs__$M z8B?Br{&=>+>s3B|mrEXANk1Ll9*Dqtf93bSz_=gX*{=L(0R{GqdlP#o)|U3x`{E{q5W67@$Cc%5`#s;!zif1_ zbsw^KCPbRFkGZM&DeU`<;!}^~ZjHkC()+P(EkAbNc0T{ZR$WtY;S+wAZtaWwF5hCi z(tO*)q>IQJf=>5c95f_S+YAy%IfWuWZrxDPKl?c$3*qh4uOwr7w5BbH)vxEp-y6*pB*Pkq7NHwE%F&pjMIb={~9-1 zUSa;^Q63td5L%A>$crj^F(ft@oAjAy83`sVMMxTHD6SXSIk#0ekJS23SwKU7= z4`-0FM-oWwPjQk?uzKN0hILKx(yDcB;c$wMUGd_Qj(uSmQ9%8f=W%oE<}1#+b5VaY z^d)f@%V0G7q`-Jpvdp4(ql2l^76_mg>L5^ml=UKyGB}5a-#Kl%_R zJH+V7C3{?Mdskd|aCgA}?ynJ2MGK2dE4=Bj3Tqpl`f*t6vqr~VHKLZa@}K3JdGd3~s)h*3(5&vz%BC=M-G*WEuU_DZ{4|0tMs$%MG!UcSkC z#S&fOb#ID~=~0%1yKSo8ZCs17EtT!FwJX&krlJrUvr?@Zhw+%}=NnkMH+8`&$;^A& zF-}2Pvao*#ITrlN>K#NW=^ht&>7$HM`jgoiU8io{qD6Zh>PsD@5R*CSYHqIcA+;k@ zVP*Y}K`!j2hxIT)?=oG0mQ1TqS&p-N3(V_)ab9CV-%of}2H%;ky?Dg_*srB7UX^!j z|4BaA3rS9+p@ed8hOk-~U%+|E{$LqFW+J%LF~o{Rm3bNM?*JXNWTbXO}CYw%^d16yp~C+PWdS?`<#_Vw)yPFp6&wVX4N9@@?G zFr&@$u@ehj<{%Yxw<#N_MinP9m(0vj95hQfxW?seC7cyY*GHo7u`D=a6E8aey4*G- za3p{8N5tC|4fMdfxXmswzxLd<;PuQaDpNe5K%*t)r6E~ z=)-r+z7CF$qsO0KSF=W)2cZh8B*G(^n5f^<#or#C!~Fy${{U~3?rQAvq<}^4i;WSE zO4hkw((Yfbu}Wsc9&^fq3^E0{{Q^q=WoKgzUmj4vgz|c*>R)W+P+%(t42f(R-?Vz^ zkI^j~y09r|ev_Q%5?VGVxu_(pfYR|%;mjx$0VH7phv_%DtOH?pyY^f{u>+i~9iOM5 z%0@QgsG{gPsQwdT6QVd9XZ4Kk4wPz;awJLX zFC&})j#^BI&R3QEOmm}{kR6N#EfHfko@2X5|$Ad-c>13T$ULxToFOg4N7%K@lKW~sf4{s@)B4d1A5 z-?R)f6;LFZ9534)kAh9<LDsJRB0B57FUP$Ay8AqE~q^on+-dET1 zzXtLWrbnb7r6OR7twrF5uaNlz2G!D(8gM6L8VRVtbYlIVei|e?u)A%RmAVs+1PKep(={Qo5 z%08jq^7tqfIazqL_pSR=Mf^SmWdVnxT?OjzSWHl)^$pjK|7~ zCdNCZP8aBT2<9~bNb`xrb~y9buux;|_cFvgW#$!14dNIxg2V4Q=_?9j2)Q#4^OTFx z>j-0OABjk)6SD&KAlw-J+_h9k7ro{Tiyc_GPOmw!H;`=@LTB7NK|oMy5YLqk4b%Jt zV2fw=!b3jxs-pU-Vk|I4YQrATo_Y17g+-D%$h$m3oPR){A^LodcIVf24>&gm4m@qbQ(>WmSGMcv{;RnOTLqLM}@*ZK#>97Q}y z5!rg;18bklns+NBX8Q-``0f4~D|U>=+*%Kq^$F9}09>WRo_}!G$EL~zC2(5p zxVf><;q^A=?vFx(jwFF+>}0>Kv)d}Omm=c%_yerWj_*fFSJ?b>QK(|(UvZNj6rYJ@DWO4x z0Z{`;-qNb`&=j6^8MX^6QcD~TVN0hOHDi!>sH3) zyH&S;hi{ikz(b>z*5YrHnViVb@63`;fVxoY0R}m2@q&XhR0s_BYG}-^@+d`r49c!efxo`*rPBlVz>x0 zKIA-&fdh_j6cAf{*x`wQWv zqz1M;zZ4RwmE|sD1Kodiw5I69yAeK>c!{{)`44UWX z+|errZltIu&1KN;BvDese1HZI4{3F_gS7xUQ~u89dF_Q86ojldCt|GaTfXlHov-;& zZN4S6P<QVDRDr$!g_e@}0wZAJHeh z#WYX=FIrWiohL|598^YP1KJU|ZgNUmVx+Vyt9ufS@$_g7TJxuK5V0gW`-Y`+0HD-N zV&{VsMaTf(ax}I}lGtBRC^vqIAf+2@r3u$Pq8tkkB_SG~`x1pl@_1^$h8vySV4IGT^4bSrWG z>P!NG9H+I?yK!JLCMsB)^e=uUXqa6RK5ZOA_%w@Kob8?0k-*KYXQe-5;d>hiMIdvF zjPEa=4d~gk?DZ4r7#mHs$j_!3nTa-YhlX^BmyCmd4q5hUD3|A|c8K7`7$-)88LK#o zNL_@B<^fP>cK9f8#{H1UKm3uNAd3Sa`9SUnbrU$SxhwU7g79;FAgLMy*dLIC6wl<^ zUIk^ovFK`p-}m#Wl&Y0blO??OBbeH(VTygk*3uXa<3oPH-2BepQ8>+@!rjWwP?jMv zO~x3Jg0ZwLe0dKeQR)7qfTMr~z#n07&bdK0_UCTXA#>!{38*vTBJ!&HY(SUnfly6( z$kJ^5!2syie-(UiRhgOi8x2|Je@6*`pJ_XU3jXf7aS7ShqL_KThWZoi7>9UFz=8l? z_{N^|;?jak%(=(cg3J$1%Ecim5ztqpalPUf;ai11#$)KK`i*lcJSgCz&&5RIN%vey zZjqLOMg-S(2H1O;AgVoimrNBzHE0ZAu-~IjDa(H0`&6%HO-%Rmx0b<~Bd=VBYU)d* z_0e!m@b0h-h1S{CpvLgafJ#!Cs^~4UCHnUJV<-((;ktstj2t*DVKuD$d8g)IOe(h! zW3;eh7YT1+u~&0)ghntbcyOHgL;)gu5W8mJaD1;q6$+cM`h@`0&YGg_`!>6+Vu!Rh zdQ~C6t>QM_pO^s4jTUPpDZ$xJps{G`-KF_pUx}X{Hz&IT3zjq-0oD_2P7s`qTcG6u zXNq85mMldQ#ixg!woZX^i?|95-TcF$#tk8c!6&Dg-B!&xpreq2C^@o{JMv@IrYBw{ zJdtvVXiPhoWjh9>=dEO6nKOK6$t#kDHcRAo%Oh70NrlB9VVS&7k)|RV^vuljhb^BN z8ctC$h!FH|Hpo>r5|0kKdss|n@H3EYB{wQiI7e-^gpih5GKQ-&M0AVp->WPwS&)ZQ z4!D++Yucj!&d`UemjC*=^iC@{%|L$IH@V8xKku{Zhyu@$O)Lk%QWjlCEh}D@Xo(PO zXmv&r=un8mhWT9y{|* zo>|Nswps;+Oennlb$a^!xPwUVh@uw?W286oeIZcrx8$cg#?GSb`1}JTzTkLFNZsgF z9Ef>eaSq$*_h2hFI>>mlefu8J{DK*G44lkQrkI%RWbDhVbYT@kp_IWwbh?{g`=&{o8hh` zJi2Y+5QT^`Y?g3(a@N0YKQM_tY~hE+vq>oIV7d_LrX&SMZ}9+{I%egI6j{hZc^B6s zoymJ~Ei0-&N&Ye!UDxZ~IA~`>(nM%p`Je7`;1<$fW&1({sLl+)H3kRV6gat3MYk;+ z7RzJ~+)rJ=B_Soj>-n%EW_IE5ovaYV+E)$vuUAn6$_Y0;8uR)5pZ`jI+lmMTfnWoX zjno>jkKl_Ohw5G235=?Lq{$#Y;p)+53M@)>wP52{h9Fk+#RD|r|H^wjavcXy_x&jF zbAu>e1n|_y(a7R3MC8_Q0V9c-;8j6L9N|vGF3v;B|y8Ywx z?EObvsB{-ET=2)=4+hoh#Ti_Wgyi^Q%W1izCx1UJGVz)E375s|{^NiS{TWt*l5T)l zrYD_$ht=UCKA%#UgFDIsA1G<8!W1}?g*gsbfE_`uz)ufLKXccCw9`nmLCDSs{5zWBTBD58i# z<*I>i7(!FGf@IO=f^VEX`er|@FA*ot%P9_{9g=vyhPz){mHXD}Z&r95v`M?& zg@A5<(N(@uM^Sh>BZjnmd0xm3Oy$v23+}R0CUpG6z`^SN~d@ zI2EbFKRAibCx678KKQz=#)nuMZNt}ddB9Ec_K)#o!hm!CfzlWzhDs*{60l-;XeG|j zatt=gsz#$WCuK?(wSTonq)~{d?&V z9#)?s3T!lp*+wN-cI3OI<)u(1ftFerx~Et1tK#nk@PaOL>FLq5RKp?BI1#DIIkE>IPaD zX02byw3ew_S^4vcO4h$?1WQ{;2EZfam88kMWo%9;3g;YxlF<&dlGc7a*=8q-6UIQ* zT^pQWE4nIc!88_j%t;biTX_Fg%Bx4E&G{iE($I6|h^l|7 z{Ci-W>N5wYRh5-GI<00-G>MOED+^&7dJ?RlBm^8;S8WS2CrZC`8k)AQWE0(Y_l7bf zpb9$(g=y^yscRE=c3^FHsfj<5G@bO-aVnV32&ueYCKp9*5;SoKCHoEXIRqBxd=1ZlE0D(lRO{(g%) z9j0OHja&!(7*v)bi)QQl$r^EGrfVm{?q|%69Ei5&f}Zy_1?AY;r48^X1$7uxQ5=oXKBUm} zO|uCu+do{MrvsyWUXkwxUaxS)A9=dqu^#+-7?;d9Vdys!BGMJ*Tis9GA-&aj;Uj_> zF=`IUIiCvi1bA?8Q!MB%5~x%T-j60)_FoP$WkGYIrE1&8{nl8tn0*64ktG6naIk); zc_H>OmqgxK3zw9nQG9`hDnSSyr8QscIn=`el`X*JKeR#qNU7O!lO{V6_?n7oMO*mu z&N6#PKywlz7~L zPiv4CKrMh8s+@9QZ2ndbw{mxmLn}_z1~mo2s>{BObbQja^w3tCAJf6{3roWS2Juc? zn$XfE)AR||483F!Q-q-n$N(qgsuQ4}rhZ(~^voi^$-V;+K3N;}Xf zVQ(%1=X|9{5#$oKzJ|#Gd6EsO;^TCVlV@|q5pQO)h=aCLsc!H+g0a5EO$TBP~ivEPK1m+^+-6-p-Kfo+-^&8gY=hMEE%-Y7I9bUXozICuAL*7|5SC zi;&qrw^|`$x+4T)H-!I{<6^+3##ki#Q&rkpg(5yL<9OBl#v~tPrtp1`=wOK#QFsuym9yNXm6 zXB}nw1XW0F#?a|l&O*`&s$<9)4+>#Tl%1Y=lp-{EmCV4C^-j0G^OLTU@IA{%1EhgR zXb$}%IlOjE^@S3!r*SpIj_Zmp>Z7w)v!QH37 zKl+C0`Togis~~&s&D7|0Uv7%)|P8v z0WxXcB(|oi4>49Z)4Q{Y;%k#WU_%BmQwUdiL7>>J+`lS)Vt2Jd{^+vi(G{Nv=STy!L{r zYA;p<$Zy;g6sLFLTJc9CB>MhQ ztz^ikx2x@J^Rr3-#aB)qyPMC7H+BX_vtAELeqjlh_Q<&49xWs@a4YRdo&mfKa!d8( z4wJdr1}w-4)21%{6fDIm>NgtjjLT)wWdNCrd7z)ge1*&qyt_<`sPI1EcKQg$vn`kZM8GV}Xd7eAfj~FUMB!Qr>+@(}Nfnf{I{# z&e=v>Z+TqX<=>mMg8c(uKEy|1TlXk`ZuALhX)EWdR6+=r28B=B_K-{QS&C>j>_GIt zE7h@NYb9Wzom&g*+g@#X(JOhKV5J}tUPo21-@QPZZ$)5Wuob<#CXFwW zGl??RmxY&oJ8OEDw^Jn4_p2NPp0->CR=k$(_l(krMIPz7b(WluDaAky&WLC#c5$}^lClHt%!|< zdO!Phz_gGpfiqe$&hA)STHS&OB{Y9gG$0y7R#Gvw!lY%Z@LN0$JEW_#IiheV7Fj4? z5yahgGm~X3@xW+w&4#*{VQ+*cmLCy>Ki0(w>|dRrX`6O$dhD2gKl$ovoZ|ScqRC4O z!Vsd*=InU16!C!$ms0Fh1zt_`UH3F-LhyS&alrrsCJ9dF0JJ*3#3jDLpkr}4yC#Sz zz-pnz_9bK>3@67Q!*hQ9>_oLF-Q6KkYkHZnKpTzcc8R?jF94c0i?CDm}Gbn%T=L8q%7f56k0#f5Y_u*U3pD5IKuJF$|{WV z_!u{(RVn+N7|o7{{x8)1$Ko@o$va$H#!~|#;!9>@J{W7Qt1>uWzQ)pb>C7}A&rzS$ z<$)tOaD)qe)h5BnSQLzAVw%jRg=yOyoQhQcp-eL~frC5zvk@KHSrxV!&S9Skpo0bs zIIdN^YNyEEY&|&q_euKNFo_czKICAfQ+i@)59w39zvn#mS7`lVF_->Djf4a~p`}nCTO>u|nZhojO=6lTTWxw4l5T3yP zbYl5Yb8l@qAQ~F(>bbo}&*U|-pnZ*F(N<13d1MSMP4Oup* zkdLR$Looyr+$R^;o0mSfFk10kdVqquC@(&q@N0mz%3QpafTKhb$resRX9HUg_&&ob zf7IO-cge8pv%_{)s1>PM8iU;**HVLRDld`2b&(7%#^4JGGcPMx{d9ldN1b5VgmX=ntv# z;;TR%n3>;#Dx(OY=h?kwd9+gsANFl$+}8U`=P2s>Aupt@_Jvy$-^nC-HqMg+;YrfUsRqdYc;Lw6vqrAp!!zmLT`VGXSOar z4v#egC7@sn*CY3KxU z*oa`ztSFye9ubroj{9AEL&lZ9Yg91<#DF~rNZ|DgmK6Q|Q-sG6*VMy1j1PyBDJBxA zJf`L^>-0#e=uD%M?&;@@1BT`EZg(T_SUg3OUL(X5QNTuDcSFvfQVO#`(*{WkQ!XW# zUbw4!>C7eD{2>53|3Hb${J6Q_>Qd=`+?oBrmsn*M4Qq#X`kFBcu@Vcwhw@UEz;zg4ms!XtU)Fry6fn=o};SUir2Ijm&V^a!Aq1A~O4 zG-0qU^a=H#M6Y0mCMRWQCFQ!9X`8~Y2X=k3+@}9^zXI}AjVk_ff|X^M745LHX6j-> z)L!Ki8%2}Qu&OH;VY6RV)q0(jbVS`ST=oc8e*2S&coi|^q`=LvgS3;bD1X?cOkhqa zODaT7#C;Sv6sMP}@4Kt#4BAdNBL9^^jf@^OjGV%smW-I}p21U4hPRKPor*c7o3$tx z(|qygJIEKf!9p?fmEI%4^NuH0Gfeg6PmlD31XYM0n~7!vVDogOe`3=> zzZPMfWfdi1_Q6mwh@e0uIbr|+2|!jtRKtDsJln%NY4QG3`0y`Pd>usqvnREy*00sB>Xr94%`Gve;5)0R5^r(okv;Z`gRR6Lxe=!!#qouiE-kbwq_RFlW_ zjsmB^-QfeIJU4{c4M=BhA${|^CfwycB~bg(kY#TBp6GC}f%z0X-$sbR@4&@9Hc9Vu z8w;D%J>^mRPb!WLe!0Nuc_RhGk+$t2`fuR^->017O8JgOMIkHcCDdt85r!@nNg~6p zUvO7sCEWg&xPG3MFHi{5g&XdZ-p&gQIaUh?IC%Q51O0~b*5ps_rArjabv;3-{bLYV zRr`Uf)d$rQcZ1$t!qC0bkvEgt#V=8FYk{y@;pK9F;jpJ_VL2$aE|YQnSTNp8j#aIb zaFhwxq{btmDk zg3?*veaVdL!(mgl+5giy7l;{^lxoS6ebrL{$lk+{^0m*~)F5({hvgG)M%6e*ykd!X z*K8!&IQ&fthz|Ly!ur%1UrT4`q>i}z8%5=0*bpiiC~%ZQ{a`umi4r_=2=5%DI_X)L zHMl4^P+Fs;iW%@m_GhpN-C*)@q%h$!l2oEDn1GFX(j=nVxh3i-!V)eK4PLvrAOnW( z$A|b3ojl7%l7coF2+auYuX(=?Q`v?&BgM9Ry%d)u0vCijx|x(*G$}A8p7Nw6LpcJp zzxT5WEUQY!iJ3JNHYFa4PCY$0=t!?#B9O)Rf`hTb?-2mfoWKq z5Rl!M@jprVL?n^$t%KUL!b77t1%M2{G?Ip?@#Kf`zF0b!uwk>utiQSY^P}&e9g~wG z>ZfE~@PkToa9Dq_?hu+mB^Hu8{)BMUe*!%eQMcS%j^i`*drcPHR3!PNfW#o>d9scw z_?+PUr=uvny$$=i)vHif7YwdgFwZ7anhAoz`iU31a?&b5{dmLsak831j<)B!;3Ef( z)blkS_P2mhiHMz*VdN)BvZO!{w(yY<@#do{_wpOTQx90h6p4RpRCGD# ziIdN)#7%F`V07gqR5VvOY%f{YL4in2#xU@UePr&!yDoR^1VU(~_ONSMK&I=9rhJ0l zZ=Z0>xp#&!?b_PupIEz@jGZq@P@FjJKIcy~_Wjz# zG;ng^Ax&pzYk>Fb_wB13)BeZK@YlL*eTTK}s zbJEY{@0-M0Mi3o5^{`slbM)>VRsF6rS)&JczU86t(p@0PN(8guO};Lx!)8NH&yBqO zJf;5@PbENzYvT`keQjr|7ulsJwVoS$XEf&+%vfK+M*l5a=13^-H&JTpj`C&>E>04J z0Thsd8793naF|IvNHe8$iQwetmMkAl;Nm_R!~7|iZCLEIye^xR`x;S%;g^N$VG;I@ zC0J=8p<^bv`Gp#zqawQW=cOpnFTR9AQ9k9VA7@c?1NGp}TDJ&nQ4RLJ?(}@UVH~;X zy2?5(_`X>jRPO!-d&epEkS|yS`MlP>A;I)A&w3Z0SHq5x0ygGD!>qw z@I2|=UK)wV6@HBwH!s>+B$^5<|MmJ<#0eLW$Hw$0)oU)J3PJSfxcmx&^Go-PL|>N0 zG#k3bQXCbu^UVm`lh%r%rzFIXsO$1J@KOkpty}4$J4>mQyM+ZWd6_XuM2<9+jZS_| zSR#u>nq=y`yMMO2AhPbPclkwp-a&^Ix%}VzebEm{01~c$|GOfWADUJsrFyza`p=L@ zvOoCf#Uq(J&0obQ)yfDOaP;C)?vPVEst||f@jV|5o(TIA*pDVqA~Z-M(rP4#)?!nu zke-T8dLqxE@Kq?j>YqpO;+rg=rZP)&K5W}q`xVLt>vV?N@VYnfjZ&C2%YjOo?aF6WDZC$xG%OfD@k2!}K&)q26O(SswNcgrH zCNo9dJrZdbwZ7AVU7 zIlIw#@8@##k?QCNdZ0Ay!RZ_7lkDgVl$hXA6yWr>GzRyRQqUsCtcFz=4^`1{c=|re zD3->eF&vyv=gQRq!S$o{`^Vq)5*KOYnVyei+8244*B|@FL=`+r3zVfvX(a=lw~&1mW4%nlmb{E{PzlCXJt2?sZz?qsf9wETA{nh2PyqVP{u((pEl^Wt`d%h z_KYom@e6z=&<}f244?@Rd;N`w?wA73k^?}0Ewj?gX=n{!F{7Lei@uw`=a~% z??zHD5DNkeWB*N}T#HhWkv2ab1*kB5mO`pduQS}Aa;%Arm7yxwyBqsaXhQm*V zOY=DTMDw4F-ORN4s4YP6@M46{OdO!#n@t1Gn^3|{vb%zxrl&p`18LRn6Y3fKYo=Hz z($#e5J=e%^A(__KxoPMF=JO>XVUcwMV@vwc?NJP7{f?cjS)?@Kh9SazJL9Fezrya+ zYnW-tI-7@>qtP!jt@Q;SQ-UJ78Glrn6`Kow=X~uMdAa=LbV|7M!1fi0f@_Ey9rflM zF&Uw@`Rm74`iyd4;#{%To3bq_a#yptC8D<=Xy)nJt5uk<*)@hoMn zk2{O;Mn5XdepHxvgGS42ej9Yxd~t|P_)>X!f2q!LTf<*7A3oNc03@6ycNs8`C-QoC4;nE*;#x@~$H?IB-}y@_n1xvJU;glTO0-Sblf0 z5Zc_GqT+I0SQGYn8h=&7jp;=va*s<^qC zR+#g4X&wNPGyFaAB$0CSPVtxP$w1jK$1(leyzEK!IN4Bqz|NR>L#@y%gZRhQsqwk- zgG{ zDKDV)-Y7&J}JBj_bIufr}9Q*7g;Z(}Mop4mgQE|D0fQdDtL%4a76WnZ+}2ZU`)# zM;pQWHH+xtjh+7g&>ksfmGXrZ?W6R_3l!{kXpv^5XvH;L{1*Nd&^Sw@06xa7epmkcJh6_umW&P85S0mJHt9fZIU&e2K&5q&pf* zH+wnVnfxDPSXC)tlN!k_Oi)jD{;|Kea`s9=oIROoGN(h8K>-W|n5&;jX8Xvfp=B@@lr_nbKx77c&`4eB+l+*yLJv~1T!rj0(k{v$$D6GV$rmA8} zZ(t%Z`9HU!0v0~**8f&8=S%J%zvZY9NQ@w=mD4RBYvNV))Q6OZ?Q}>!uH{kP1)=sp zjGweCI>`#!twODl zSdU zJC%||OxR-1iwvOGxNmG4fA~1oq&u~W%TGVyz~FyUho`fqxQ?2P#AFo2LG11eU@{^F zUs~2Qm(bZ7Oir+UeANdPgm1Lfv8lM<9rCjv4&DS1lV`$kdzDM7%#s(EXy;DUht z^Evc(EFSYQ(_jlibK=&kUxUWhs>$h)OnW?h4_&^5IGb>{& zQ*APH1@oP#bPtmpDP4$&D@)B)InCk;wquvCf9!pApVUahID->zY=yl4xkEt6pQLBg z{Y(MS|McRPv64m;^#+?El+vH<;a!QHXk9I%r7LVumx3Ts2T3|HyCk8L8ZK#0t3b_K zNcBa5Kis}{!@Q=fYATC4kP?>E!aV)MQ=o)cIoLk1UXb`QK}05Y=(4aCP0T zJLczvGXli@bGa@Dn!_QL`a$nuv3oRZ3Q2IY~XM*LOTzOYJEBpdOF^y+52I3#8tr*%+s1)HsbZhuwe ze>}-b#0>AN3PKNJf0A|O>sg2GJpZD*4b>% zyoftI_$xD8t&F2Lm50dK^Jb1Gx+>m}iHLD-An6MrfgEklzZ%EaHPX3yiCWmW@@w1K z?SXO5gid7WR4ir8eUw8Q;CU$F@;Dk|BBT(y5$&-NC$M_s3N8|5CRbeX8Id?Ar-ww! zE`Qrf5PauVI?b@LAJ34J2&;`INGMZ7?V?8!XNB>!@%}qJjIN5rkJV^3oekXn)mJB3 z;6H?2&AZFwU;bK<^sPeQ3b=HHAsnF2Qxj&W(YHL*tS$v57cC~UWG*8OT>>pqe*PS3 zLNnBp=;MgX+=3p7Qc3O|mT$H|eiT3aEXj9075XY-*Fl&Hgw>>p8;$;wbpnQVd`@~k zjW_W%H>UdggPy;zJNyv<0UxUC-!fTkd7v7mKqS=$s)Lf&g__7$Pyn9)cEy*x zXk(FLbt_qiOr;Uko_mtU4Ls5Xbii=mAI6Xf$kGYo=2|k?#u=WQO#tDQ@N301`8J@) zK)ic9ulO)Fc+3nZNz`)(cJ(L_btWr+fQ7Q=6dog<48e=yP<7XnbLB8S(No2xI; z|DBK^XyU&~YyO;%j*#jQ4E*9e5_X>2nuhL3gZu05I^14h{ml#Jg1aonY(E5OxK0A) zi1q?Z+LJ?-kzpYh0r;*e2&3D!o6GH}ETk1I?QHFwLNeoQZ8++$9*=V6I z7;B}T6Q5>qo%-H7dAyjT#N$yF3MbA-q4VV%JN1uk)4IncrQ8d~C+VZ*N3$_k|By#U z&XG^p!#>{XdV*CUG|N2j99YTsEnj#WsppoM$U(_gx2HpGFwY$=Q2wWZsa1_uvpS(x z#2+`aN3(QPaX%h^)WB*{YuEWV564Uj>$i=(+o=W=WdE!J7vJ);{A#2&w|2G*31Kdk z#m%CH#0Iw$1q8L?pCOW?KmSWY9*m$rAp&@&f`|i$9b+QDLD6?Q*z!US$gk|V_=Uy; zGp-RSU-aE~6i>f!Fv!oxCR>;7K?bulhQIARlW{w!8kuX6ttu!>Tpt50R!~mz^1_2g zFIr#0*_EuMH^nVbQ*|(O({gKKI}=QzV<+#aZff{eo{%Fw4BIwa$Y7XNnyAI{>*9xE zPWgHkzm-$?VGRCouLGKdPi$}xcs{<*b!qxz<-pKOe9e!&4%_IA1QW_bt1k^#5uP^U zChg!D{y7U?>#G8!Y1lCN&(#Qt;ITy?ZaLO@lzM^uK84Xaq6r*85HgUJgcG_`f_T=| z{1sbnJ2GAPEFThOc}fDKEj9dRV@46yseW zr(43;IL1FZ)R*5ZR{cK%zJ23y_ZrMsmmo9LN)H7K+ngc~bO8au)W34#bBXN*vq4RL zl-s5Ds6L($(-62Ry2s1QD_W;7s3`jRfgLDks z*SpsJ5AIs?^UQhII`f`q@6X<+(_dp-Ze5V;X#y7J?JOLH-Gs~BjiK$~AqYH!Aknyk zqeCi;sIzrg@~T5Ep9=ZASSR3-LWn$^!qAIf^7npG)q1ym$jWl)SvdHDZHQ>yqWfjh z^rjXPF@d9NJsgk}!fBmFlM8L580Yn@soYzu@YJF2jeBp5UKy_aOcUxq=el`r>NhAm zExCRB`3lSzSd?+@_#1%_Sp@dcH~eeC1)=5$KEDE7xe+u=K`(aPd~rc4U$N(5jbOFkR z5rc;`Sl&bZ-I`kKk`ucSI0ZY#oV#&ThF9R zTg)e$MG4lT5V@(}Eb*q{pY4fos1+#`ujMnxz9YR>6_>_1EG7?Xg?3UrzbVFXDYMybP zhKBtyAp9d03K7l77YiXO*_4U)=cr`n_Vx#fOuF-h-i2Yni@e~R!xd6T#uOt=zVJ$c02V!JESlWyDSc&DQYCQO4p6GNbzLgVi^df zo}&9i57mbwQGpi{`?)B{G`%oY;_iweSFb|W(b6c}Yeck)C;#!E=e7G)h@lfJBJcij z@t9>GSZ@IfjtR8Gw?Q~WYYgE_gI8=HUTmlsgAcM3Z0-m=WuPP^Ck5DRQ zunF6%F2=e~IkxGhfq|hJ_;Sf8sJFCm2eiwubp^Kb?LOA`zi`brOzl+_oZZnV5#x!| zn(yVyO)M=DgJUR;t*AK^b@y$!JCN;-qvN-k5GhiI>PdI_$MEzwv3JmiA%#HqM|==giZYwngp7i^j}9>_wKW%QTB zR6oif5VA+dOfxVzXBFktH<{E`*tvabKpHcn%KiLqg|iZ>ox<)&DHs$ie6M9%CE`}8 zorv078{Std7T0q?DIO)W%38|XjH_X$Bvmmn5xc=3?qN)S$RUnWz)N`e<6$eJa9;&O z^y{3H`JoHpO$UifW%+uFQ^1_LE3PunhUM?ZAZ~#T+Kx8PEYT0J15Tninw9xK3~!jE zilUGa4*W!~79Yr>H>2*eiJDq(YL2_@iZA|{m4(6W7}m_wFQjNH7*;Mw?MOq^07AEy z%D`lu;am>07Nc5L!Mb=Ki!YR#6{M0NMs8<;jLLElhE&QkKdMqK$QOf zvH)%`KK!mbmsZO4iu-(~#O_;5=os|1&CqEDcOu?SY5BFy0BQAy-}4PmCwR;e)p=BK zCY{k2!4qnIiNhmI8dm52^uDl`rXe?11elJv|9;j8`RANvav`@Y;#Zo4T>r3?I{m62 zL%W~K&9tmd>2R9w57-Vzh>MI^taxWzG7&K+C>oQmbvVn&)zM=>L7Ta+{vW`*W#4xl z%ZCuXRA@=-ISBK5dSkG9_OiDEG6r!v68a?FcgVt|BiNjf%*ktsm=`P;pqw|M}u7QJziz{t%f;=am zBNcfURZF2K;kU=#acX8|mU~L@ymxA}2~&iZSYY;ZLws~2hz~(l-bb)sknWGijrAZ0 zmqsB3BM%y1Y)EQYKvb$Chkl2GjB!U>HJaI_&3_)Sw<#(uQmO}Zd^`%QO;HFZrw+}P ze)FC$RjSmncshYQ3fgwzbIyascjJ}sXJs*^Fbb+yn-~f*9rVf3e40sDC*O>B{F2B4 zJE{@8G&L&YMhudzB?G#3c~7PxNlA{sYph@d`ww}4Xy=_J7BAOwFlvypDeShZvNcM! zAc?WpK#;`*U6_M>un>c*`fVOVG#LnQXM|Dq|fLXvyC9`<+N($rPqvLVAAlc^M{8>M!9au%h0bK3lr6U!Z?SeW% z@jX=PaR}KKXN>M=+_wbF{s^cr*H6nYNeISguR9R~llIJ*cdmq}Rq=yNt9TzwUk=AP zQk(|br=Yen9fW}!hg3s|wpOlT>n@BWrczyCU8Fm4sI(G-w2TmNn3^GYst=%yQFV)J0wJg7E84^`f%Q*@8NFTCc)L8G}ez3Pzjv}e3 z7q|SmFjFH((Q5+35m3#oR=Fe69l<-F#>c*zwM10k^5}NEl3EY^_l1qmp8$BOIr$r# zk?q?-z!^&^T2nO!m)`vOrz_oVtx@Agtqm8w`1cLx#md;bCxRq#Mc6Zs=OsyTZ4~T9 zlv}!aO<6MQSoP8K_Ua|*0R-5V-7+GRSz#>tf=IA6AM5TNn<(%v4~ zXTk7*vivzdzog(W7(7ii+thJL-w|}jgYsF=xvQ(>(xawYgM?M^tLId3M3M#Pz;aN@ zXNOe5lIbvoejIwzp6eLui|VMt;%~B>gf=GIdiSmGD7N~NcfsuS5Dsf@l+)-r*JSV4 zd=85gACP``%=+>-$O|@Syh^jtgei5@)f0^me;WKZIvNKrxK<%$Q<3Ib4d@~Bo4<6T zdCtzMFt5*euqpylu~@ZtQt`Ra&BN%g+k*TTb2^YQHBPf9GDDNUZcJ@_Ou?~spUTi3 zk2m1LA&NKW*g@3`^Bk5wYT zMGe0C3c6EW`|v|xM#LpvB`j!LZF=;vJqSaT4dvg8D`@4(=5@!UTu3U*JOPThjjnC@ zQRLO|13&pP^@2#qkC`oxxh@>DDEg!8lWb&#g!HpNapZ{ZClNdXH9PTc73?2rG3H0A|F_TAY7acBHtOylgQi0*-~-vvHE!R zz7XKHIgmH>Oqm(z(511RaGK_c`tEy=r0%CLioay@`20D9NX!d&W%vHp5aQ^Hj*6}! zt$<&y6U68U3!|Gnw*9=j>q=`db&KuTgF4Gm38hwR(hYZ0AWn2!3`(k}AXtvN+CgAA0e^SpQQgsV-K z{6X6#twjyF!#qeLqscb-p_;8H(gwZ8X-*)ft95vbBf`hdh|)MDHK=vu;fg z*xsqXI9qWJofe5INmt7`+i{>?^g4OEw^0%K!^r4f3H|XatuZbHi@5vlA?kuO%22{- zV;XzpKxUvVxy$E+d1iUwK}`cvN}0|})Zu*nXf4!$Cu6%et}Q22!SS0eR@Kp85@=xQ zCw`(x%awyL+sNC1KLc{#*+1iwuXOU&C71D;`5tl}eMoB8V4WtRi$WZn*N@Z(=S~lb z8MoICy9f@u%`fwc=H>4k0|u&?nc)b|YtF+Ljf=l!^>si8S;aGLxE?-jIN!>>O=8Fd zZ%<2`jq7c2C*IFtpI-5yv5cwoynkLbvU3fc@71pxi}X(-H|LpVRnLHo89BA?ykoj! zB*FdLQ#xj=`_rDoX9jc``K1`o|LJ&S*~L6*t_}ZgtN29n{!IY!#+9`J73j6K<_2<} zvq6Y=lXL24xY(1!?~jzOQ1@ET>q67{wzwj|hbQ446jmz|uixt=nO%Ek?b$`QA7^Uv z%=dcwb?!mC(N~#}_h+KsBR|>WRWV_bH|lLW@P>?b#NW*?$=>IBpaMQG53d&gR2;6y z?w;d5wi|Q1xE`1TBmb0%(yx^g4(WWR3aPs4zwxG~OGms=@mJ!jgZ9s}`cT?)^oR^$ zbeMCSoftnIX-hFdRkE=D6`oG!0zP`IO#9*xQ7xImro398Z)o?#6=lSH4ZP-CAcB|_ z`X8p$=Nvh$lpgomtLzV(GLLZgo=>zfJ2_^xvE_3b;!cD%7sk1jzlsrW(>i}23ZcJnb-#;~{?MDd@p+}p{}V%Gc= z8O-reo24{>r4WwnKxTS2`_iLt8^kYG?DKvqABov3qk#s6_OUbOXeDWUvDwAjB^VR8 z`7#35imn(C*dAm6lI%%(RsuF3CWl%m#^bhrvdKR4yGI{0KJbjdDEfH*Uff18PIoK( zC>1gL1LWUXo;4|X&koX@!tz{6bfVI2j5v3dz@rcPlE;#IHo#G)R?7sEyCYVotu$Ar zFLAcU?Xqx2Ti^Fll?wW2gx?4LI<;o5BM?hgN}#?j1{=>U2oaTox&h8{s6^s7SF3SY z5!P6oe0IlCHVR&Pr2sf8LUCg6B|Ipt2%%6XzT^70^puHIM8sNAD_XBCE+o;!B!YLs zNN+cr#24@1Oirxy*`{lt(OpRnf8ytV`qV48IvwT7^P=jbS>s=pnj92Wn$n^}I|7BM z+v7y`H0CX9Fe33wCgmJ((v5LN56aRXTUdaHx$xRXbW4u1<&QYOXTDnK$kd-73>7Em zV!DnCn5qzAKbJ2!LiFOB-6|1gIJ=$AgI`vl#!;4Ur14YT{XCEB;M*e;tPU$uO$1m- zme#w<`Wr;24#<PB9(+fkD8gt$R8>FDsxUQGW|WML)l3eV0v+ z`g$NArP#x$h9ZWy#N>Aa&v1?}NU`#I%1Xe*wS-Hx+NFC&iz7kmor3(xAVP;5b_{xW zt9LD5F$dKbPe?HCbxI>5bxcWmdq|$>9hgPnO;f0 zRD@SI>G<>Ql7aMVT7&Ww_h=?MyB`%7W$UdZY6L^Xetw zcM%k0R%q7iJMbv)D)ANL^hlet|XrpGr=rLnnuMbwQV!u+%|2HXe9ZWN;2y*-y11p;;~%A&rIh5Gj~ z=#F!3T6p;+Y}aL&9GJmuumF6VI#@1jlY~1O%S*d@$4=ORiz{nyo2M6KF^I>*0rlB3eFzj?Z>?zX^-O|s ze6v}0YRYqln{6F%0v2)HlsVOKg02?D+36NzH5$&q2*5I92ji$k|GrxgW8GNA+7O8T zEV$Wd)p%(sB@l(-*iryMN30M%=|3cKorP_6U%@&vZeM2YoUemY$-Fwo+9j~6e=B=Ig5)-{ zz+m(6esR~x?OED|u}(*Mfs|K$id>!30$@%2B0Sh>VO^clsHjLO^odzOkCG9MhyU?A z=iO+FNnze5o8>E#c-$=*|FLCJ*)x7*<5&=4$w6}X)j3s26}l$P#C;&MiH+*$7c+4C z*s{*@0-(bATryI>Q)hbH)KzH{Ccj~IF89nsfJ-#|IoTt)DXrFobZ>I ztUqTVrZ%*KJ0xeZ7zt-ffLxth53n06f}d@@kINdz)Dr&_*i*m@q?J@eNFW(yslL1< zF8CAN!RTzJUj{FN{pR;cs|~KLovs%irO<@jfkdO!(KUc*UahP_8~DSqvd zr~qr6=b1FF3NHo=nJwpU-^57&ygaT%E>)B5i;zKphC$CatR5+&uT=TpyOL>5CG>5} z1f`fubHdpdAMoW`ShY&KQlQlkpKopUUp-O6T?r4hv)OXQ@}33~9+w#)n&ZxC){l|c zfHL*82M_oKtq+OC? zuD*DN*MT;`#O9NlwEz)cXNyh>7YH-NEJ|J1R$~}1<{WsT>*m<56C4rU%t=7|mY_q) zT$Furv5AmfA80>(7y3XgD_DLm+;seBiR}NfC}%@ zbo)d=UfIuPiR0>8qJ6)7LSp2rq@P*{I;yka|=)B|ZOb7#Z zEf7Y0>m9+|WU&MO&arw5G&b#uAd$1r+1f$U@{nXw+vRPJyYB{C!*BT~`c_s89qvlZ z4pZpPTAx}v4!XSf+ydW*^Q-soOK+rJ{D&R$JyPTgWR1_xud2Ud*;i3nm(Ux*2FEz9 zk-3AKuc&Vtd{3`)!MIVcmDfdV)dBFC>!(^n>ab2}jI>~hH|i_h>K`1Yz>Up7Ar~?} z?BL_+-OiPAE|hb$^7q8!KOQX0*bB{L?H)PAuH~sH@whq={7`@_jB6@%;dcl3Kl_H@ zsNzVZ{OL2P8)YH^22Lu9WA`a=%TZQ1SDJ%WX`9NO{q@$S7ZtBBkWHP3WVs&t?Yo>?^u6uVS%X6|r_23nzSWcx%kYJmJVbQ-pS@E>QHHAja@Ovr zKDS%zfvq2s;e;4(_LIz!?tubRE|c-@7jKUrN3Tul&A7>DoQ3jHCtEDy`!>eCbeX47bf+R>` z5=5YH-@@Lv1~U8OlDbP=ltC3!8;vRoKAZs4Li42{pp`&;@1s<=&>n_EL>y2NL~BGO z^d%i{)B3|ywd2q`^OB5kPPiVL?uC@X02CZ z1f{~GKl%osI^)ci|3cyA+3v-hzGDu-gCPw0MNxQ0yuR|KsAkWBnn`^>&?kgK`vU^o zx&CHKCw$BGa#?u5rRa)MH454wz+Wj(@-y~Tt$ZFHkqUGdqKsXGv3N{1B2vWP2L6{? zv}BpHKyRZ_Qo6SDswIetd&6?=Yq~L#fJx9P^3RyE>(Mb_l_`D>yz%{47_mKcB|Rem z-F==BlzLfedU0GPMe+F?XrAs;F(oq|Tq;N&y2pr$+ZL_qY+F?QvS^`R0CJuWWZWARce%PH@$T^ZVd*`S9Vam>Mzmws+eSdwyQpcP+3$(XDS47^ff!y{o8~t z?qe9a=RFkuS&mqOIK zFs{9exNv4dGemh{1FAP^>+Ce8(dp%CW6v#YaRrOesjxSmVUp1cq+19c|?@6N-)Pzby(zSGQjJ> zq_70XOg4_Ts^Yml4u>|sjv(9xj8P4D@0@S-6fUT7?$UoK+2So$R3io{2IT5Q42WBs z&1jSE{}VBN=|yUmizQ1RERRbVn$M~b*v|pp9SM?>A?^`ggqIgl zH=ZlsRv$Sej`A>z@aq%H*N6@P?UEtS!%0x07iF=GIBvIHNCHc@U77HaMK2So9_dV) zqJx89+H9;pHjq6Y`vG(bypDS0^H=CeotRvhHdqGvzslIX*Lj_uuPoM2fG$J)|M-bO b-acg}i(z0b(wPD+P0(8fb@?h;v#|dG{M_8J diff --git a/providers/google/docs/operators/cloud/dataplex.rst b/providers/google/docs/operators/cloud/dataplex.rst index 28cfe48ee766b..f409770ef89e5 100644 --- a/providers/google/docs/operators/cloud/dataplex.rst +++ b/providers/google/docs/operators/cloud/dataplex.rst @@ -15,19 +15,19 @@ specific language governing permissions and limitations under the License. -Google Dataplex Operators -========================= +Google Knowledge Catalog Operators +================================== -Dataplex is an intelligent data fabric that provides unified analytics +Knowledge Catalog is an intelligent data fabric that provides unified analytics and data management across your data lakes, data warehouses, and data marts. -For more information about the task visit `Dataplex production documentation `__ +For more information about the task visit `Knowledge Catalog production documentation `__ Create a Task ------------- Before you create a dataplex task you need to define its body. -For more information about the available fields to pass when creating a task, visit `Dataplex create task API. `__ +For more information about the available fields to pass when creating a task, visit `Knowledge Catalog create task API. `__ A simple task configuration can look as followed: @@ -40,6 +40,9 @@ A simple task configuration can look as followed: With this configuration we can create the task both synchronously & asynchronously: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateTaskOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateTaskOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -59,6 +62,9 @@ To delete a task you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteTaskOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteTaskOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -72,6 +78,9 @@ To list tasks you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexListTasksOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogListTasksOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -85,6 +94,9 @@ To get a task you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetTaskOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetTaskOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -109,7 +121,7 @@ Create a Lake Before you create a dataplex lake you need to define its body. -For more information about the available fields to pass when creating a lake, visit `Dataplex create lake API. `__ +For more information about the available fields to pass when creating a lake, visit `Knowledge Catalog create lake API. `__ A simple task configuration can look as followed: @@ -123,6 +135,9 @@ With this configuration we can create the lake: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateLakeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateLakeOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -136,6 +151,9 @@ To delete a lake you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteLakeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteLakeOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex.py :language: python :dedent: 4 @@ -145,8 +163,8 @@ To delete a lake you can use: Create or update a Data Quality scan ------------------------------------ -Before you create a Dataplex Data Quality scan you need to define its body. -For more information about the available fields to pass when creating a Data Quality scan, visit `Dataplex create data quality API. `__ +Before you create a Knowledge Catalog Data Quality scan you need to define its body. +For more information about the available fields to pass when creating a Data Quality scan, visit `Knowledge Catalog create data quality API. `__ A simple Data Quality scan configuration can look as followed: @@ -160,6 +178,9 @@ With this configuration we can create or update the Data Quality scan: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateOrUpdateDataQualityScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateOrUpdateDataQualityScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -173,6 +194,9 @@ To get a Data Quality scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataQualityScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetDataQualityScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -188,6 +212,9 @@ To delete a Data Quality scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteDataQualityScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteDataQualityScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -197,17 +224,20 @@ To delete a Data Quality scan you can use: Run a Data Quality scan ----------------------- -You can run Dataplex Data Quality scan in asynchronous modes to later check its status using sensor: +You can run Knowledge Catalog Data Quality scan in asynchronous modes to later check its status using sensor: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexRunDataQualityScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogRunDataQualityScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_quality_operator] :end-before: [END howto_dataplex_run_data_quality_operator] -To check that running Dataplex Data Quality scan succeeded you can use: +To check that running Knowledge Catalog Data Quality scan succeeded you can use: :class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexDataQualityJobStatusSensor`. @@ -232,6 +262,9 @@ To get a Data Quality scan job you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataQualityScanResultOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetDataQualityScanResultOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -249,9 +282,9 @@ Also for this action you can use operator in the deferrable mode: Create a zone ------------- -Before you create a Dataplex zone you need to define its body. +Before you create a Knowledge Catalog zone you need to define its body. -For more information about the available fields to pass when creating a zone, visit `Dataplex create zone API. `__ +For more information about the available fields to pass when creating a zone, visit `Knowledge Catalog create zone API. `__ A simple zone configuration can look as followed: @@ -265,6 +298,9 @@ With this configuration we can create a zone: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateZoneOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateZoneOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -278,6 +314,9 @@ To delete a zone you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteZoneOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteZoneOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -287,9 +326,9 @@ To delete a zone you can use: Create an asset --------------- -Before you create a Dataplex asset you need to define its body. +Before you create a Knowledge Catalog asset you need to define its body. -For more information about the available fields to pass when creating an asset, visit `Dataplex create asset API. `__ +For more information about the available fields to pass when creating an asset, visit `Knowledge Catalog create asset API. `__ A simple asset configuration can look as followed: @@ -303,6 +342,9 @@ With this configuration we can create the asset: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateAssetOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateAssetOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -316,6 +358,9 @@ To delete an asset you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteAssetOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteAssetOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dq.py :language: python :dedent: 4 @@ -325,8 +370,8 @@ To delete an asset you can use: Create or update a Data Profile scan ------------------------------------ -Before you create a Dataplex Data Profile scan you need to define its body. -For more information about the available fields to pass when creating a Data Profile scan, visit `Dataplex create data profile API. `__ +Before you create a Knowledge Catalog Data Profile scan you need to define its body. +For more information about the available fields to pass when creating a Data Profile scan, visit `Knowledge Catalog create data profile API. `__ A simple Data Profile scan configuration can look as followed: @@ -340,6 +385,9 @@ With this configuration we can create or update the Data Profile scan: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateOrUpdateDataProfileScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateOrUpdateDataProfileScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 @@ -353,6 +401,9 @@ To get a Data Profile scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetDataProfileScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 @@ -368,6 +419,9 @@ To delete a Data Profile scan you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteDataProfileScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteDataProfileScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 @@ -377,17 +431,20 @@ To delete a Data Profile scan you can use: Run a Data Profile scan ----------------------- -You can run Dataplex Data Profile scan in asynchronous modes to later check its status using sensor: +You can run Knowledge Catalog Data Profile scan in asynchronous modes to later check its status using sensor: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexRunDataProfileScanOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogRunDataProfileScanOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 :start-after: [START howto_dataplex_run_data_profile_operator] :end-before: [END howto_dataplex_run_data_profile_operator] -To check that running Dataplex Data Profile scan succeeded you can use: +To check that running Knowledge Catalog Data Profile scan succeeded you can use: :class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexDataProfileJobStatusSensor`. @@ -412,6 +469,9 @@ To get a Data Profile scan job you can use: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanResultOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetDataProfileScanResultOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_dp.py :language: python :dedent: 4 @@ -419,22 +479,24 @@ To get a Data Profile scan job you can use: :end-before: [END howto_dataplex_get_data_profile_job_operator] -Google Dataplex Catalog Operators -================================= +Google Knowledge Catalog Entry Operators +======================================== -Dataplex Catalog provides a unified inventory of Google Cloud resources, such as BigQuery, and other resources, -such as on-premises resources. Dataplex Catalog automatically retrieves metadata for Google Cloud resources, -and you bring metadata for third-party resources into Dataplex Catalog. +Knowledge Catalog provides a unified inventory of Google Cloud resources, such as BigQuery, and other resources, +such as on-premises resources. Knowledge Catalog automatically retrieves metadata for Google Cloud resources, +and you bring metadata for third-party resources into Knowledge Catalog. -For more information about Dataplex Catalog visit `Dataplex Catalog production documentation `__ +For more information about Knowledge Catalog visit `Knowledge Catalog production documentation `__ .. _howto/operator:DataplexCatalogCreateEntryGroupOperator: Create an EntryGroup -------------------- -To create an Entry Group in specific location in Dataplex Catalog you can +To create an Entry Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryGroupOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryGroupOperator``. For more information about the available fields to pass when creating an Entry Group, visit `Entry Group resource configuration. `__ A simple Entry Group configuration can look as followed: @@ -448,6 +510,8 @@ A simple Entry Group configuration can look as followed: With this configuration you can create an Entry Group resource: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryGroupOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryGroupOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -460,8 +524,10 @@ With this configuration you can create an Entry Group resource: Delete an EntryGroup -------------------- -To delete an Entry Group in specific location in Dataplex Catalog you can +To delete an Entry Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDeleteEntryGroupOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteEntryGroupOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -474,9 +540,11 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDe List EntryGroups ---------------- -To list all Entry Groups in specific location in Dataplex Catalog you can +To list all Entry Groups in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogListEntryGroupsOperator`. This operator also supports filtering and ordering the result of the operation. +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogListEntryGroupsOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -489,8 +557,10 @@ This operator also supports filtering and ordering the result of the operation. Get an EntryGroup ----------------- -To retrieve an Entry Group in specific location in Dataplex Catalog you can +To retrieve an Entry Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGetEntryGroupOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetEntryGroupOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -503,8 +573,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGe Update an EntryGroup -------------------- -To update an Entry Group in specific location in Dataplex Catalog you can +To update an Entry Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUpdateEntryGroupOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogUpdateEntryGroupOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -517,8 +589,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUp Create an EntryType -------------------- -To create an Entry Type in specific location in Dataplex Catalog you can +To create an Entry Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryTypeOperator``. For more information about the available fields to pass when creating an Entry Type, visit `Entry Type resource configuration. `__ A simple Entry Group configuration can look as followed: @@ -532,6 +606,8 @@ A simple Entry Group configuration can look as followed: With this configuration you can create an Entry Type resource: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -544,8 +620,10 @@ With this configuration you can create an Entry Type resource: Delete an EntryType -------------------- -To delete an Entry Type in specific location in Dataplex Catalog you can +To delete an Entry Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDeleteEntryTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteEntryTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -558,9 +636,11 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDe List EntryTypes ---------------- -To list all Entry Types in specific location in Dataplex Catalog you can +To list all Entry Types in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogListEntryTypesOperator`. This operator also supports filtering and ordering the result of the operation. +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogListEntryTypesOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -573,8 +653,10 @@ This operator also supports filtering and ordering the result of the operation. Get an EntryType ----------------- -To retrieve an Entry Group in specific location in Dataplex Catalog you can +To retrieve an Entry Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGetEntryTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetEntryTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -587,8 +669,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGe Update an EntryType -------------------- -To update an Entry Type in specific location in Dataplex Catalog you can +To update an Entry Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUpdateEntryTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogUpdateEntryTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -601,8 +685,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUp Create an AspectType -------------------- -To create an Aspect Type in specific location in Dataplex Catalog you can +To create an Aspect Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateAspectTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateAspectTypeOperator``. For more information about the available fields to pass when creating an Aspect Type, visit `Aspect Type resource configuration. `__ A simple Aspect Group configuration can look as followed: @@ -616,6 +702,8 @@ A simple Aspect Group configuration can look as followed: With this configuration you can create an Aspect Type resource: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateAspectTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateAspectTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -628,8 +716,10 @@ With this configuration you can create an Aspect Type resource: Delete an AspectType -------------------- -To delete an Aspect Type in specific location in Dataplex Catalog you can +To delete an Aspect Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDeleteAspectTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteAspectTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -642,9 +732,11 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDe List AspectTypes ---------------- -To list all Aspect Types in specific location in Dataplex Catalog you can +To list all Aspect Types in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogListAspectTypesOperator`. This operator also supports filtering and ordering the result of the operation. +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogListAspectTypesOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -657,8 +749,10 @@ This operator also supports filtering and ordering the result of the operation. Get an AspectType ----------------- -To retrieve an Aspect Group in specific location in Dataplex Catalog you can +To retrieve an Aspect Group in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGetAspectTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetAspectTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -671,8 +765,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGe Update an AspectType -------------------- -To update an Aspect Type in specific location in Dataplex Catalog you can +To update an Aspect Type in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUpdateAspectTypeOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogUpdateAspectTypeOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -685,8 +781,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUp Create an Entry --------------- -To create an Entry in specific location in Dataplex Catalog you can +To create an Entry in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryOperator``. For more information about the available fields to pass when creating an Entry, visit `Entry resource configuration. `__ A simple Entry configuration can look as followed: @@ -700,6 +798,8 @@ A simple Entry configuration can look as followed: With this configuration you can create an Entry resource: :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogCreateEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogCreateEntryOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -712,8 +812,10 @@ With this configuration you can create an Entry resource: Delete an Entry --------------- -To delete an Entry in specific location in Dataplex Catalog you can +To delete an Entry in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDeleteEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogDeleteEntryOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -726,9 +828,11 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogDe List Entries ------------ -To list all Entries in specific location in Dataplex Catalog you can +To list all Entries in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogListEntriesOperator`. This operator also supports filtering and ordering the result of the operation. +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogListEntriesOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -741,8 +845,10 @@ This operator also supports filtering and ordering the result of the operation. Get an Entry ------------ -To retrieve an Entry in specific location in Dataplex Catalog you can +To retrieve an Entry in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGetEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogGetEntryOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -755,8 +861,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogGe Update an Entry --------------- -To update an Entry in specific location in Dataplex Catalog you can +To update an Entry in a specific Knowledge Catalog location you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUpdateEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogUpdateEntryOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -769,8 +877,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogUp Look up a single Entry ---------------------- -To look up a single Entry by name using the permission on the source system in Dataplex Catalog you can +To look up a single Entry by name using the permission on the source system in Knowledge Catalog you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogLookupEntryOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogLookupEntryOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python @@ -783,8 +893,10 @@ use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogLo Search Entries -------------- -To search for Entries matching the given query and scope in Dataplex Catalog you can +To search for Entries matching the given query and scope in Knowledge Catalog you can use :class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCatalogSearchEntriesOperator` +The executable example below still imports the compatibility name shown above. +The preferred alias for new code is ``KnowledgeCatalogSearchEntriesOperator``. .. exampleinclude:: /../../google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py :language: python diff --git a/providers/google/docs/operators/cloud/dataproc.rst b/providers/google/docs/operators/cloud/dataproc.rst index 039a0e8e4afd1..115b1001eb934 100644 --- a/providers/google/docs/operators/cloud/dataproc.rst +++ b/providers/google/docs/operators/cloud/dataproc.rst @@ -15,15 +15,15 @@ specific language governing permissions and limitations under the License. -Google Cloud Dataproc Operators -=============================== +Google Managed Service for Apache Spark Operators +================================================= -Dataproc is a managed Apache Spark and Apache Hadoop service that lets you +Managed Service for Apache Spark is a managed Apache Spark and Apache Hadoop service that lets you take advantage of open source data tools for batch processing, querying, streaming and machine learning. -Dataproc automation helps you create clusters quickly, manage them easily, and +Managed Spark automation helps you create clusters quickly, manage them easily, and save money by turning clusters off when you don't need them. -For more information about the service visit `Dataproc production documentation `__ +For more information about the service visit `Managed Spark production documentation `__ Prerequisite Tasks ------------------ @@ -83,22 +83,22 @@ in that environment before relying on the corresponding Airflow operator in a pr Create a Cluster ---------------- -When you create a Dataproc cluster, you have the option to choose Compute Engine as the deployment platform. -In this configuration, Dataproc automatically provisions the required Compute Engine VM instances to run the cluster. +When you create a Managed Spark cluster, you have the option to choose Compute Engine as the deployment platform. +In this configuration, Managed Spark automatically provisions the required Compute Engine VM instances to run the cluster. The VM instances are used for the main node, primary worker and secondary worker nodes (if specified). -These VM instances are created and managed by Compute Engine, while Dataproc takes care of configuring the software and +These VM instances are created and managed by Compute Engine, while Managed Spark takes care of configuring the software and orchestration required for the big data processing tasks. By providing the configuration for your nodes, you describe the configuration of primary and secondary nodes, and status of a cluster of Compute Engine instances. Configuring secondary worker nodes, you can specify the number of workers and their types. By enabling the Preemptible option to use Preemptible VMs (equivalent to Spot instances) for those nodes, you -can take advantage of the cost savings provided by these instances for your Dataproc workloads. +can take advantage of the cost savings provided by these instances for your Managed Spark workloads. The primary node, which typically hosts the cluster main and various control services, does not have the Preemptible option because it's crucial for the primary node to maintain stability and availability. Once a cluster is created, the configuration settings, including the preemptibility of secondary worker nodes, cannot be modified directly. -For more information about the available fields to pass when creating a cluster, visit `Dataproc create cluster API. `__ +For more information about the available fields to pass when creating a cluster, visit `Managed Spark create cluster API. `__ A cluster configuration can look as followed: @@ -111,24 +111,28 @@ A cluster configuration can look as followed: With this configuration we can create the cluster: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator` +The executable example below still imports the compatibility name +``DataprocCreateClusterOperator``. The preferred alias for new code is +``ManagedSparkCreateClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_hive.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_operator] :end-before: [END how_to_cloud_dataproc_create_cluster_operator] -Dataproc on GKE deploys Dataproc virtual clusters on a GKE cluster. Unlike Dataproc on Compute Engine clusters, -Dataproc on GKE virtual clusters do not include separate main and worker VMs. Instead, when you create a Dataproc on -GKE virtual cluster, Dataproc on GKE creates node pools within a GKE cluster. Dataproc on GKE jobs are run as pods on +Managed Spark on GKE deploys Managed Spark virtual clusters on a GKE cluster. Unlike Managed Spark on Compute Engine clusters, +Managed Spark on GKE virtual clusters do not include separate main and worker VMs. Instead, when you create a Managed Spark on +GKE virtual cluster, Managed Spark on GKE creates node pools within a GKE cluster. Managed Spark on GKE jobs are run as pods on these node pools. The node pools and scheduling of pods on the node pools are managed by GKE. -When creating a GKE Dataproc cluster, you can specify the usage of Preemptible VMs for the underlying compute resources. +When creating a GKE Managed Spark cluster, you can specify the usage of Preemptible VMs for the underlying compute resources. GKE supports the use of Preemptible VMs as a cost-saving measure. By enabling Preemptible VMs, GKE will provision the cluster nodes using Preemptible VMs. Or you can create nodes as Spot VM instances, which are the latest update to legacy preemptible VMs. -This can be beneficial for running Dataproc workloads on GKE while optimizing costs. +This can be beneficial for running Managed Spark workloads on GKE while optimizing costs. -To create Dataproc cluster in Google Kubernetes Engine you could pass cluster configuration: +To create Managed Spark cluster in Google Kubernetes Engine you could pass cluster configuration: .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_gke.py :language: python @@ -139,13 +143,17 @@ To create Dataproc cluster in Google Kubernetes Engine you could pass cluster co With this configuration we can create the cluster: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator` +The executable example below still imports the compatibility name +``DataprocCreateClusterOperator``. The preferred alias for new code is +``ManagedSparkCreateClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_gke.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_cluster_operator_in_gke] :end-before: [END how_to_cloud_dataproc_create_cluster_operator_in_gke] -You can also create Dataproc cluster with optional component Presto. +You can also create Managed Spark cluster with optional component Presto. To do so, please use the following configuration. Note that default image might not support the chosen optional component. If this is your case, please specify correct ``image_version`` that you can find in the @@ -157,7 +165,7 @@ If this is your case, please specify correct ``image_version`` that you can find :start-after: [START how_to_cloud_dataproc_create_cluster] :end-before: [END how_to_cloud_dataproc_create_cluster] -You can also create Dataproc cluster with optional component Trino. +You can also create Managed Spark cluster with optional component Trino. To do so, please use the following configuration. Note that default image might not support the chosen optional component. If this is your case, please specify correct ``image_version`` that you can find in the @@ -193,15 +201,19 @@ You can generate and use config as followed: Diagnose a cluster ------------------ -Dataproc supports the collection of `cluster diagnostic information `_ -like system, Spark, Hadoop, and Dataproc logs, cluster configuration files that can be used to troubleshoot a Dataproc cluster or job. +Managed Spark supports the collection of `cluster diagnostic information `_ +like system, Spark, Hadoop, and Managed Spark logs, cluster configuration files that can be used to troubleshoot a Managed Spark cluster or job. It is important to note that this information can only be collected before the cluster is deleted. For more information about the available fields to pass when diagnosing a cluster, visit -`Dataproc diagnose cluster API. `_ +`Managed Spark diagnose cluster API. `_ -To diagnose a Dataproc cluster use: +To diagnose a Managed Spark cluster use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDiagnoseClusterOperator`. +The executable example below still imports the compatibility name +``DataprocDiagnoseClusterOperator``. The preferred alias for new code is +``ManagedSparkDiagnoseClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py :language: python :dedent: 0 @@ -220,7 +232,7 @@ Update a cluster ---------------- You can scale the cluster up or down by providing a cluster config and a updateMask. In the updateMask argument you specifies the path, relative to Cluster, of the field to update. -For more information on updateMask and other parameters take a look at `Dataproc update cluster API. `__ +For more information on updateMask and other parameters take a look at `Managed Spark update cluster API. `__ An example of a new cluster config and the updateMask: @@ -233,6 +245,10 @@ An example of a new cluster config and the updateMask: To update a cluster you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocUpdateClusterOperator` +The executable example below still imports the compatibility name +``DataprocUpdateClusterOperator``. The preferred alias for new code is +``ManagedSparkUpdateClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py :language: python :dedent: 4 @@ -253,6 +269,10 @@ Starting a cluster To start a cluster you can use the :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocStartClusterOperator`: +The executable example below still imports the compatibility name +``DataprocStartClusterOperator``. The preferred alias for new code is +``ManagedSparkStartClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py :language: python :dedent: 4 @@ -265,6 +285,10 @@ Stopping a cluster To stop a cluster you can use the :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocStopClusterOperator`: +The executable example below still imports the compatibility name +``DataprocStopClusterOperator``. The preferred alias for new code is +``ManagedSparkStopClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py :language: python :dedent: 4 @@ -277,6 +301,10 @@ Deleting a cluster To delete a cluster you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`. +The executable example below still imports the compatibility name +``DataprocDeleteClusterOperator``. The preferred alias for new code is +``ManagedSparkDeleteClusterOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_hive.py :language: python :dedent: 4 @@ -294,9 +322,9 @@ You can use deferrable mode for this action in order to run the operator asynchr Submit a job to a cluster ------------------------- -Dataproc supports submitting jobs of different big data components. +Managed Spark supports submitting jobs of different big data components. The list currently includes Spark, PySpark, Hadoop, Trino, Pig, Flink and Hive. -For more information on versions and images take a look at `Cloud Dataproc Image version list `__ +For more information on versions and images take a look at `Cloud Managed Spark Image version list `__ To submit a job to the cluster you need to provide a job source file. The job source file can be on GCS, the cluster or on your local file system. You can specify a file:/// path to refer to a local file on a cluster's primary node. @@ -304,6 +332,10 @@ file system. You can specify a file:/// path to refer to a local file on a clust The job configuration can be submitted by using: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`. +The executable example below still imports the compatibility name +``DataprocSubmitJobOperator``. The preferred alias for new code is +``ManagedSparkSubmitJobOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py :language: python :dedent: 4 @@ -408,11 +440,15 @@ Example of the configuration for a Flink Job: Working with workflows templates -------------------------------- -Dataproc supports creating workflow templates that can be triggered later on. +Managed Spark supports creating workflow templates that can be triggered later on. A workflow template can be created using: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateWorkflowTemplateOperator`. +The executable example below still imports the compatibility name +``DataprocCreateWorkflowTemplateOperator``. The preferred alias for new code is +``ManagedSparkCreateWorkflowTemplateOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py :language: python :dedent: 4 @@ -422,6 +458,10 @@ A workflow template can be created using: Once a workflow is created users can trigger it using :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateWorkflowTemplateOperator`: +The executable example below still imports the compatibility name +``DataprocInstantiateWorkflowTemplateOperator``. The preferred alias for new code is +``ManagedSparkInstantiateWorkflowTemplateOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py :language: python :dedent: 4 @@ -439,6 +479,10 @@ Also for all this action you can use operator in the deferrable mode: The inline operator is an alternative. It creates a workflow, run it, and delete it afterwards: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator`: +The executable example below still imports the compatibility name +``DataprocInstantiateInlineWorkflowTemplateOperator``. The preferred alias for new code is +``ManagedSparkInstantiateInlineWorkflowTemplateOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py :language: python :dedent: 4 @@ -457,18 +501,22 @@ Also for all this action you can use operator in the deferrable mode: Create a Batch -------------- -Dataproc supports creating a batch workload. +Managed Spark supports creating a batch workload. A batch can be created using: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateBatchOperator`. +The executable example below still imports the compatibility name +``DataprocCreateBatchOperator``. The preferred alias for new code is +``ManagedSparkCreateBatchOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 :start-after: [START how_to_cloud_dataproc_create_batch_operator] :end-before: [END how_to_cloud_dataproc_create_batch_operator] -For creating a batch with Persistent History Server first you should create a Dataproc Cluster +For creating a batch with Persistent History Server first you should create a Managed Spark Cluster with specific parameters. Documentation how create cluster you can find `here `__: @@ -509,6 +557,10 @@ Get a Batch To get a batch you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocGetBatchOperator`. +The executable example below still imports the compatibility name +``DataprocGetBatchOperator``. The preferred alias for new code is +``ManagedSparkGetBatchOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 @@ -521,6 +573,10 @@ List a Batch To get a list of exists batches you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocListBatchesOperator`. +The executable example below still imports the compatibility name +``DataprocListBatchesOperator``. The preferred alias for new code is +``ManagedSparkListBatchesOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 @@ -533,6 +589,10 @@ Delete a Batch To delete a batch you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDeleteBatchOperator`. +The executable example below still imports the compatibility name +``DataprocDeleteBatchOperator``. The preferred alias for new code is +``ManagedSparkDeleteBatchOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 @@ -545,6 +605,10 @@ Cancel a Batch Operation To cancel a operation you can use: :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCancelOperationOperator`. +The executable example below still imports the compatibility name +``DataprocCancelOperationOperator``. The preferred alias for new code is +``ManagedSparkCancelOperationOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/dataproc/example_dataproc_batch.py :language: python :dedent: 4 diff --git a/providers/google/docs/operators/cloud/looker.rst b/providers/google/docs/operators/cloud/looker.rst index d9751e0388f3f..085b5f9bb230a 100644 --- a/providers/google/docs/operators/cloud/looker.rst +++ b/providers/google/docs/operators/cloud/looker.rst @@ -15,14 +15,14 @@ specific language governing permissions and limitations under the License. -Google Cloud Looker Operators -=============================== +Google Data Studio (Looker) Operators +====================================== -Looker is a business intelligence software and big data analytics platform that +Data Studio (Looker) is a business intelligence software and big data analytics platform that helps you explore, analyze and share real-time business analytics easily. -Looker has a Public API and associated SDK clients in different languages, -which allow programmatic access to the Looker data platform. +Data Studio (Looker) has a Public API and associated SDK clients in different languages, +which allow programmatic access to the Data Studio platform. For more information visit `Looker API documentation `_. @@ -39,16 +39,20 @@ To use these operators, you must do a few things: Detailed information is available for :doc:`Installation `. -* Setup a Looker connection in Airflow. You can check :doc:`apache-airflow:howto/connection` and :doc:`/connections/gcp_looker` +* Setup a Data Studio connection in Airflow. You can check :doc:`apache-airflow:howto/connection` and :doc:`/connections/gcp_looker` Start a PDT materialization job ------------------------------- -To submit a PDT materialization job to Looker you need to provide a model and view name. +To submit a PDT materialization job to Data Studio you need to provide a model and view name. The job configuration can be submitted in synchronous (blocking) mode by using: :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator`. +The executable example below still imports the compatibility name +``LookerStartPdtBuildOperator``. The preferred alias for new code is +``DataStudioStartPdtBuildOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/looker/example_looker.py :language: python :dedent: 4 @@ -60,6 +64,10 @@ Alternatively, the job configuration can be submitted in asynchronous mode by us :class:`~airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` and :class:`~airflow.providers.google.cloud.sensors.looker.LookerCheckPdtBuildSensor`. +The executable example below still imports the compatibility name +``LookerStartPdtBuildOperator``. The preferred alias for new code is +``DataStudioStartPdtBuildOperator``. + .. exampleinclude:: /../../google/tests/system/google/cloud/looker/example_looker.py :language: python :dedent: 4 @@ -67,4 +75,4 @@ Alternatively, the job configuration can be submitted in asynchronous mode by us :end-before: [END cloud_looker_async_start_pdt_sensor] There are more arguments to provide in the jobs than the examples show. -For the complete list of arguments take a look at Looker operator arguments at :class:`airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` +For the complete list of arguments take a look at Data Studio operator arguments at :class:`airflow.providers.google.cloud.operators.looker.LookerStartPdtBuildOperator` diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml index aff958b5429aa..42553e3f60daf 100644 --- a/providers/google/provider.yaml +++ b/providers/google/provider.yaml @@ -310,10 +310,6 @@ integrations: - /docs/apache-airflow-providers-google/operators/cloud/compute_ssh.rst logo: /docs/integration-logos/Compute-Engine.png tags: [gcp] - - integration-name: Google Data Proc - external-doc-url: https://cloud.google.com/dataproc/ - logo: /docs/integration-logos/Google-Data-Proc.png - tags: [gcp] - integration-name: Google Dataflow external-doc-url: https://cloud.google.com/dataflow/ how-to-guide: @@ -326,7 +322,7 @@ integrations: - /docs/apache-airflow-providers-google/operators/cloud/datafusion.rst logo: /docs/integration-logos/Google-Data-Fusion.png tags: [gcp] - - integration-name: Google Dataplex + - integration-name: Google Knowledge Catalog external-doc-url: https://cloud.google.com/dataplex/ how-to-guide: - /docs/apache-airflow-providers-google/operators/cloud/dataplex.rst @@ -342,7 +338,7 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst tags: [gcp] - - integration-name: Google Dataproc + - integration-name: Google Managed Service for Apache Spark external-doc-url: https://cloud.google.com/dataproc/ how-to-guide: - /docs/apache-airflow-providers-google/operators/cloud/dataproc.rst @@ -439,7 +435,7 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst tags: [gcp] - - integration-name: Google Looker + - integration-name: Google Data Studio external-doc-url: https://cloud.google.com/looker logo: /docs/integration-logos/Cloud-Looker.png how-to-guide: @@ -515,18 +511,20 @@ operators: - integration-name: Google Data Fusion python-modules: - airflow.providers.google.cloud.operators.datafusion - - integration-name: Google Dataplex + - integration-name: Google Knowledge Catalog python-modules: - airflow.providers.google.cloud.operators.dataplex + - airflow.providers.google.cloud.operators.knowledge_catalog - integration-name: Google Dataprep python-modules: - airflow.providers.google.cloud.operators.dataprep - integration-name: Google Dataproc Metastore python-modules: - airflow.providers.google.cloud.operators.dataproc_metastore - - integration-name: Google Dataproc + - integration-name: Google Managed Service for Apache Spark python-modules: - airflow.providers.google.cloud.operators.dataproc + - airflow.providers.google.cloud.operators.managed_spark - integration-name: Google Datastore python-modules: - airflow.providers.google.cloud.operators.datastore @@ -612,7 +610,7 @@ operators: - airflow.providers.google.cloud.operators.vertex_ai.generative_model - airflow.providers.google.cloud.operators.vertex_ai.feature_store - airflow.providers.google.cloud.operators.vertex_ai.ray - - integration-name: Google Looker + - integration-name: Google Data Studio python-modules: - airflow.providers.google.cloud.operators.looker - integration-name: Google Cloud Dataform @@ -662,10 +660,10 @@ sensors: - integration-name: Google Dataprep python-modules: - airflow.providers.google.cloud.sensors.dataprep - - integration-name: Google Dataplex + - integration-name: Google Knowledge Catalog python-modules: - airflow.providers.google.cloud.sensors.dataplex - - integration-name: Google Dataproc + - integration-name: Google Managed Service for Apache Spark python-modules: - airflow.providers.google.cloud.sensors.dataproc - integration-name: Google Dataproc Metastore @@ -692,7 +690,7 @@ sensors: - integration-name: Google Display&Video 360 python-modules: - airflow.providers.google.marketing_platform.sensors.display_video - - integration-name: Google Looker + - integration-name: Google Data Studio python-modules: - airflow.providers.google.cloud.sensors.looker - integration-name: Google Cloud Dataform @@ -778,7 +776,7 @@ hooks: - integration-name: Google Data Fusion python-modules: - airflow.providers.google.cloud.hooks.datafusion - - integration-name: Google Dataplex + - integration-name: Google Knowledge Catalog python-modules: - airflow.providers.google.cloud.hooks.dataplex - integration-name: Google Dataprep @@ -787,7 +785,7 @@ hooks: - integration-name: Google Dataproc Metastore python-modules: - airflow.providers.google.cloud.hooks.dataproc_metastore - - integration-name: Google Dataproc + - integration-name: Google Managed Service for Apache Spark python-modules: - airflow.providers.google.cloud.hooks.dataproc - integration-name: Google Datastore @@ -899,7 +897,7 @@ hooks: - airflow.providers.google.cloud.hooks.vertex_ai.prediction_service - airflow.providers.google.cloud.hooks.vertex_ai.feature_store - airflow.providers.google.cloud.hooks.vertex_ai.ray - - integration-name: Google Looker + - integration-name: Google Data Studio python-modules: - airflow.providers.google.cloud.hooks.looker - integration-name: Google Cloud Dataform @@ -957,10 +955,10 @@ triggers: - integration-name: Google Data Fusion python-modules: - airflow.providers.google.cloud.triggers.datafusion - - integration-name: Google Dataplex + - integration-name: Google Knowledge Catalog python-modules: - airflow.providers.google.cloud.triggers.dataplex - - integration-name: Google Dataproc + - integration-name: Google Managed Service for Apache Spark python-modules: - airflow.providers.google.cloud.triggers.dataproc - integration-name: Google Cloud Storage (GCS) @@ -1330,7 +1328,7 @@ connection-types: type: ["string", "null"] format: password - hook-class-name: airflow.providers.google.cloud.hooks.looker.LookerHook - hook-name: "Google Looker" + hook-name: "Google Data Studio" connection-type: gcp_looker extra-links: diff --git a/providers/google/src/airflow/providers/google/cloud/operators/knowledge_catalog.py b/providers/google/src/airflow/providers/google/cloud/operators/knowledge_catalog.py new file mode 100644 index 0000000000000..a6b34177dc35d --- /dev/null +++ b/providers/google/src/airflow/providers/google/cloud/operators/knowledge_catalog.py @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TypeAlias + +from airflow.providers.google.cloud.operators.dataplex import ( + DataplexCatalogCreateAspectTypeOperator, + DataplexCatalogCreateEntryGroupOperator, + DataplexCatalogCreateEntryOperator, + DataplexCatalogCreateEntryTypeOperator, + DataplexCatalogDeleteAspectTypeOperator, + DataplexCatalogDeleteEntryGroupOperator, + DataplexCatalogDeleteEntryOperator, + DataplexCatalogDeleteEntryTypeOperator, + DataplexCatalogGetAspectTypeOperator, + DataplexCatalogGetEntryGroupOperator, + DataplexCatalogGetEntryOperator, + DataplexCatalogGetEntryTypeOperator, + DataplexCatalogListAspectTypesOperator, + DataplexCatalogListEntriesOperator, + DataplexCatalogListEntryGroupsOperator, + DataplexCatalogListEntryTypesOperator, + DataplexCatalogLookupEntryOperator, + DataplexCatalogSearchEntriesOperator, + DataplexCatalogUpdateAspectTypeOperator, + DataplexCatalogUpdateEntryGroupOperator, + DataplexCatalogUpdateEntryOperator, + DataplexCatalogUpdateEntryTypeOperator, + DataplexCreateAssetOperator, + DataplexCreateLakeOperator, + DataplexCreateOrUpdateDataProfileScanOperator, + DataplexCreateOrUpdateDataQualityScanOperator, + DataplexCreateTaskOperator, + DataplexCreateZoneOperator, + DataplexDeleteAssetOperator, + DataplexDeleteDataProfileScanOperator, + DataplexDeleteDataQualityScanOperator, + DataplexDeleteLakeOperator, + DataplexDeleteTaskOperator, + DataplexDeleteZoneOperator, + DataplexGetDataProfileScanOperator, + DataplexGetDataProfileScanResultOperator, + DataplexGetDataQualityScanOperator, + DataplexGetDataQualityScanResultOperator, + DataplexGetTaskOperator, + DataplexListTasksOperator, + DataplexRunDataProfileScanOperator, + DataplexRunDataQualityScanOperator, +) + +KnowledgeCatalogCreateTaskOperator: TypeAlias = DataplexCreateTaskOperator +KnowledgeCatalogDeleteTaskOperator: TypeAlias = DataplexDeleteTaskOperator +KnowledgeCatalogListTasksOperator: TypeAlias = DataplexListTasksOperator +KnowledgeCatalogGetTaskOperator: TypeAlias = DataplexGetTaskOperator +KnowledgeCatalogCreateLakeOperator: TypeAlias = DataplexCreateLakeOperator +KnowledgeCatalogDeleteLakeOperator: TypeAlias = DataplexDeleteLakeOperator +KnowledgeCatalogCreateOrUpdateDataQualityScanOperator: TypeAlias = ( + DataplexCreateOrUpdateDataQualityScanOperator +) +KnowledgeCatalogGetDataQualityScanOperator: TypeAlias = DataplexGetDataQualityScanOperator +KnowledgeCatalogDeleteDataQualityScanOperator: TypeAlias = DataplexDeleteDataQualityScanOperator +KnowledgeCatalogRunDataQualityScanOperator: TypeAlias = DataplexRunDataQualityScanOperator +KnowledgeCatalogGetDataQualityScanResultOperator: TypeAlias = DataplexGetDataQualityScanResultOperator +KnowledgeCatalogCreateOrUpdateDataProfileScanOperator: TypeAlias = ( + DataplexCreateOrUpdateDataProfileScanOperator +) +KnowledgeCatalogGetDataProfileScanOperator: TypeAlias = DataplexGetDataProfileScanOperator +KnowledgeCatalogDeleteDataProfileScanOperator: TypeAlias = DataplexDeleteDataProfileScanOperator +KnowledgeCatalogRunDataProfileScanOperator: TypeAlias = DataplexRunDataProfileScanOperator +KnowledgeCatalogGetDataProfileScanResultOperator: TypeAlias = DataplexGetDataProfileScanResultOperator +KnowledgeCatalogCreateZoneOperator: TypeAlias = DataplexCreateZoneOperator +KnowledgeCatalogDeleteZoneOperator: TypeAlias = DataplexDeleteZoneOperator +KnowledgeCatalogCreateAssetOperator: TypeAlias = DataplexCreateAssetOperator +KnowledgeCatalogDeleteAssetOperator: TypeAlias = DataplexDeleteAssetOperator +KnowledgeCatalogCreateEntryGroupOperator: TypeAlias = DataplexCatalogCreateEntryGroupOperator +KnowledgeCatalogGetEntryGroupOperator: TypeAlias = DataplexCatalogGetEntryGroupOperator +KnowledgeCatalogDeleteEntryGroupOperator: TypeAlias = DataplexCatalogDeleteEntryGroupOperator +KnowledgeCatalogListEntryGroupsOperator: TypeAlias = DataplexCatalogListEntryGroupsOperator +KnowledgeCatalogUpdateEntryGroupOperator: TypeAlias = DataplexCatalogUpdateEntryGroupOperator +KnowledgeCatalogCreateEntryTypeOperator: TypeAlias = DataplexCatalogCreateEntryTypeOperator +KnowledgeCatalogGetEntryTypeOperator: TypeAlias = DataplexCatalogGetEntryTypeOperator +KnowledgeCatalogDeleteEntryTypeOperator: TypeAlias = DataplexCatalogDeleteEntryTypeOperator +KnowledgeCatalogListEntryTypesOperator: TypeAlias = DataplexCatalogListEntryTypesOperator +KnowledgeCatalogUpdateEntryTypeOperator: TypeAlias = DataplexCatalogUpdateEntryTypeOperator +KnowledgeCatalogCreateAspectTypeOperator: TypeAlias = DataplexCatalogCreateAspectTypeOperator +KnowledgeCatalogGetAspectTypeOperator: TypeAlias = DataplexCatalogGetAspectTypeOperator +KnowledgeCatalogListAspectTypesOperator: TypeAlias = DataplexCatalogListAspectTypesOperator +KnowledgeCatalogUpdateAspectTypeOperator: TypeAlias = DataplexCatalogUpdateAspectTypeOperator +KnowledgeCatalogDeleteAspectTypeOperator: TypeAlias = DataplexCatalogDeleteAspectTypeOperator +KnowledgeCatalogCreateEntryOperator: TypeAlias = DataplexCatalogCreateEntryOperator +KnowledgeCatalogGetEntryOperator: TypeAlias = DataplexCatalogGetEntryOperator +KnowledgeCatalogListEntriesOperator: TypeAlias = DataplexCatalogListEntriesOperator +KnowledgeCatalogSearchEntriesOperator: TypeAlias = DataplexCatalogSearchEntriesOperator +KnowledgeCatalogLookupEntryOperator: TypeAlias = DataplexCatalogLookupEntryOperator +KnowledgeCatalogUpdateEntryOperator: TypeAlias = DataplexCatalogUpdateEntryOperator +KnowledgeCatalogDeleteEntryOperator: TypeAlias = DataplexCatalogDeleteEntryOperator diff --git a/providers/google/src/airflow/providers/google/cloud/operators/looker.py b/providers/google/src/airflow/providers/google/cloud/operators/looker.py index 775dfed31c83f..71e1e6897d116 100644 --- a/providers/google/src/airflow/providers/google/cloud/operators/looker.py +++ b/providers/google/src/airflow/providers/google/cloud/operators/looker.py @@ -15,11 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This module contains Google Cloud Looker operators.""" +"""This module contains Google Cloud Looker operators and Data Studio aliases.""" from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeAlias from airflow.providers.common.compat.sdk import AirflowException from airflow.providers.google.cloud.hooks.looker import LookerHook @@ -103,3 +103,6 @@ def execute(self, context: Context) -> str: def on_kill(self): if self.materialization_id and self.cancel_on_kill: self.hook.stop_pdt_build(materialization_id=self.materialization_id) + + +DataStudioStartPdtBuildOperator: TypeAlias = LookerStartPdtBuildOperator diff --git a/providers/google/src/airflow/providers/google/cloud/operators/managed_spark.py b/providers/google/src/airflow/providers/google/cloud/operators/managed_spark.py new file mode 100644 index 0000000000000..6b21ffa69bd46 --- /dev/null +++ b/providers/google/src/airflow/providers/google/cloud/operators/managed_spark.py @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TypeAlias + +from airflow.providers.google.cloud.operators.dataproc import ( + DataprocCancelOperationOperator, + DataprocCreateBatchOperator, + DataprocCreateClusterOperator, + DataprocCreateWorkflowTemplateOperator, + DataprocDeleteBatchOperator, + DataprocDeleteClusterOperator, + DataprocDiagnoseClusterOperator, + DataprocGetBatchOperator, + DataprocInstantiateInlineWorkflowTemplateOperator, + DataprocInstantiateWorkflowTemplateOperator, + DataprocListBatchesOperator, + DataprocStartClusterOperator, + DataprocStopClusterOperator, + DataprocSubmitJobOperator, + DataprocUpdateClusterOperator, +) + +ManagedSparkCreateClusterOperator: TypeAlias = DataprocCreateClusterOperator +ManagedSparkDeleteClusterOperator: TypeAlias = DataprocDeleteClusterOperator +ManagedSparkStartClusterOperator: TypeAlias = DataprocStartClusterOperator +ManagedSparkStopClusterOperator: TypeAlias = DataprocStopClusterOperator +ManagedSparkCreateWorkflowTemplateOperator: TypeAlias = DataprocCreateWorkflowTemplateOperator +ManagedSparkInstantiateWorkflowTemplateOperator: TypeAlias = DataprocInstantiateWorkflowTemplateOperator +ManagedSparkInstantiateInlineWorkflowTemplateOperator: TypeAlias = ( + DataprocInstantiateInlineWorkflowTemplateOperator +) +ManagedSparkSubmitJobOperator: TypeAlias = DataprocSubmitJobOperator +ManagedSparkUpdateClusterOperator: TypeAlias = DataprocUpdateClusterOperator +ManagedSparkDiagnoseClusterOperator: TypeAlias = DataprocDiagnoseClusterOperator +ManagedSparkCreateBatchOperator: TypeAlias = DataprocCreateBatchOperator +ManagedSparkDeleteBatchOperator: TypeAlias = DataprocDeleteBatchOperator +ManagedSparkGetBatchOperator: TypeAlias = DataprocGetBatchOperator +ManagedSparkListBatchesOperator: TypeAlias = DataprocListBatchesOperator +ManagedSparkCancelOperationOperator: TypeAlias = DataprocCancelOperationOperator diff --git a/providers/google/src/airflow/providers/google/get_provider_info.py b/providers/google/src/airflow/providers/google/get_provider_info.py index d0122a94c511f..db71b35681b74 100644 --- a/providers/google/src/airflow/providers/google/get_provider_info.py +++ b/providers/google/src/airflow/providers/google/get_provider_info.py @@ -262,12 +262,6 @@ def get_provider_info(): "logo": "/docs/integration-logos/Compute-Engine.png", "tags": ["gcp"], }, - { - "integration-name": "Google Data Proc", - "external-doc-url": "https://cloud.google.com/dataproc/", - "logo": "/docs/integration-logos/Google-Data-Proc.png", - "tags": ["gcp"], - }, { "integration-name": "Google Dataflow", "external-doc-url": "https://cloud.google.com/dataflow/", @@ -283,7 +277,7 @@ def get_provider_info(): "tags": ["gcp"], }, { - "integration-name": "Google Dataplex", + "integration-name": "Google Knowledge Catalog", "external-doc-url": "https://cloud.google.com/dataplex/", "how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst"], "tags": ["gcp"], @@ -304,7 +298,7 @@ def get_provider_info(): "tags": ["gcp"], }, { - "integration-name": "Google Dataproc", + "integration-name": "Google Managed Service for Apache Spark", "external-doc-url": "https://cloud.google.com/dataproc/", "how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst"], "logo": "/docs/integration-logos/Cloud-Dataproc.png", @@ -439,7 +433,7 @@ def get_provider_info(): "tags": ["gcp"], }, { - "integration-name": "Google Looker", + "integration-name": "Google Data Studio", "external-doc-url": "https://cloud.google.com/looker", "logo": "/docs/integration-logos/Cloud-Looker.png", "how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/looker.rst"], @@ -541,8 +535,11 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.operators.datafusion"], }, { - "integration-name": "Google Dataplex", - "python-modules": ["airflow.providers.google.cloud.operators.dataplex"], + "integration-name": "Google Knowledge Catalog", + "python-modules": [ + "airflow.providers.google.cloud.operators.dataplex", + "airflow.providers.google.cloud.operators.knowledge_catalog", + ], }, { "integration-name": "Google Dataprep", @@ -553,8 +550,11 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.operators.dataproc_metastore"], }, { - "integration-name": "Google Dataproc", - "python-modules": ["airflow.providers.google.cloud.operators.dataproc"], + "integration-name": "Google Managed Service for Apache Spark", + "python-modules": [ + "airflow.providers.google.cloud.operators.dataproc", + "airflow.providers.google.cloud.operators.managed_spark", + ], }, { "integration-name": "Google Datastore", @@ -672,7 +672,7 @@ def get_provider_info(): ], }, { - "integration-name": "Google Looker", + "integration-name": "Google Data Studio", "python-modules": ["airflow.providers.google.cloud.operators.looker"], }, { @@ -738,11 +738,11 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.sensors.dataprep"], }, { - "integration-name": "Google Dataplex", + "integration-name": "Google Knowledge Catalog", "python-modules": ["airflow.providers.google.cloud.sensors.dataplex"], }, { - "integration-name": "Google Dataproc", + "integration-name": "Google Managed Service for Apache Spark", "python-modules": ["airflow.providers.google.cloud.sensors.dataproc"], }, { @@ -778,7 +778,7 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.marketing_platform.sensors.display_video"], }, { - "integration-name": "Google Looker", + "integration-name": "Google Data Studio", "python-modules": ["airflow.providers.google.cloud.sensors.looker"], }, { @@ -883,7 +883,7 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.hooks.datafusion"], }, { - "integration-name": "Google Dataplex", + "integration-name": "Google Knowledge Catalog", "python-modules": ["airflow.providers.google.cloud.hooks.dataplex"], }, { @@ -895,7 +895,7 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.hooks.dataproc_metastore"], }, { - "integration-name": "Google Dataproc", + "integration-name": "Google Managed Service for Apache Spark", "python-modules": ["airflow.providers.google.cloud.hooks.dataproc"], }, { @@ -1044,7 +1044,7 @@ def get_provider_info(): ], }, { - "integration-name": "Google Looker", + "integration-name": "Google Data Studio", "python-modules": ["airflow.providers.google.cloud.hooks.looker"], }, { @@ -1120,11 +1120,11 @@ def get_provider_info(): "python-modules": ["airflow.providers.google.cloud.triggers.datafusion"], }, { - "integration-name": "Google Dataplex", + "integration-name": "Google Knowledge Catalog", "python-modules": ["airflow.providers.google.cloud.triggers.dataplex"], }, { - "integration-name": "Google Dataproc", + "integration-name": "Google Managed Service for Apache Spark", "python-modules": ["airflow.providers.google.cloud.triggers.dataproc"], }, { @@ -1553,7 +1553,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.google.cloud.hooks.looker.LookerHook", - "hook-name": "Google Looker", + "hook-name": "Google Data Studio", "connection-type": "gcp_looker", }, ], diff --git a/providers/google/tests/system/google/cloud/dataplex/example_dataplex.py b/providers/google/tests/system/google/cloud/dataplex/example_dataplex.py index efe6d6c848599..0982620b86fa0 100644 --- a/providers/google/tests/system/google/cloud/dataplex/example_dataplex.py +++ b/providers/google/tests/system/google/cloud/dataplex/example_dataplex.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that shows how to use Dataplex. +Example Airflow DAG that shows how to use Knowledge Catalog. """ from __future__ import annotations @@ -89,7 +89,7 @@ DAG_ID, start_date=datetime.datetime(2021, 1, 1), schedule="@once", - tags=["example", "dataplex"], + tags=["example", "knowledge-catalog"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py index 5e1539cb8076c..3dad30b30e87f 100644 --- a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py +++ b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_catalog.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that shows how to use Dataplex Catalog. +Example Airflow DAG that shows how to use Knowledge Catalog Catalog. """ from __future__ import annotations diff --git a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dp.py b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dp.py index c88a576ba1b3c..5d315b67ffc1d 100644 --- a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dp.py +++ b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dp.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that shows how to use Dataplex Scan Data. +Example Airflow DAG that shows how to use Knowledge Catalog Scan Data. """ from __future__ import annotations @@ -134,7 +134,7 @@ DAG_ID, start_date=datetime(2021, 1, 1), schedule="@once", - tags=["example", "dataplex", "data_profile"], + tags=["example", "knowledge-catalog", "data_profile"], ) as dag: create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET) create_table_1 = BigQueryCreateTableOperator( diff --git a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dq.py b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dq.py index 0d9956a9e46fc..b35de66a52f6e 100644 --- a/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dq.py +++ b/providers/google/tests/system/google/cloud/dataplex/example_dataplex_dq.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that shows how to use Dataplex Scan Data. +Example Airflow DAG that shows how to use Knowledge Catalog Scan Data. """ from __future__ import annotations @@ -158,7 +158,7 @@ DAG_ID, start_date=datetime(2021, 1, 1), schedule="@once", - tags=["example", "dataplex", "data_quality"], + tags=["example", "knowledge-catalog", "data_quality"], ) as dag: create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET) create_table_1 = BigQueryCreateTableOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch.py index 9f87cbc91e9c8..a3fdb3c7dfa47 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG for Dataproc batch operators. +Example Airflow DAG for Managed Spark batch operators. """ from __future__ import annotations @@ -66,7 +66,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "managed-spark"], ) as dag: # [START how_to_cloud_dataproc_create_batch_operator] create_batch = DataprocCreateBatchOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py index 0c8ee3cbe4c98..444f90edb388f 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_deferrable.py @@ -60,7 +60,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "batch", "deferrable"], + tags=["example", "managed-spark", "batch", "deferrable"], ) as dag: # [START how_to_cloud_dataproc_create_batch_operator_async] create_batch = DataprocCreateBatchOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py index 59a55075ffe97..c13f4e0062c28 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG for Dataproc batch operators. +Example Airflow DAG for Managed Spark batch operators. """ from __future__ import annotations @@ -84,7 +84,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "batch", "persistent"], + tags=["example", "managed-spark", "batch", "persistent"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py index 121c2f2c52e0a..a9bb93be98e94 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py @@ -68,7 +68,11 @@ } with DAG( - DAG_ID, schedule="@once", start_date=datetime(2024, 1, 1), catchup=False, tags=["dataproc", "example"] + DAG_ID, + schedule="@once", + start_date=datetime(2024, 1, 1), + catchup=False, + tags=["managed-spark", "example"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py index a77e15723c2c6..94fd0b2264dcc 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py @@ -93,7 +93,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "deferrable"], + tags=["example", "managed-spark", "deferrable"], ) as dag: # [START how_to_cloud_dataproc_create_cluster_operator_async] create_cluster = DataprocCreateClusterOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py index 91d688c5c3609..d1bb17ad4da86 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_diagnose.py @@ -72,7 +72,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "diagnose", "cluster"], + tags=["example", "managed-spark", "diagnose", "cluster"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py index c83190d2444b2..d7c639a1af03e 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_generator.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG testing Dataproc +Example Airflow DAG testing Managed Spark operators for managing a cluster and submitting jobs. """ @@ -92,7 +92,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "managed-spark"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py index 0a8b28b18b827..e21f0195c2ced 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_start_stop.py @@ -67,7 +67,11 @@ } with DAG( - DAG_ID, schedule="@once", start_date=datetime(2024, 1, 1), catchup=False, tags=["dataproc", "example"] + DAG_ID, + schedule="@once", + start_date=datetime(2024, 1, 1), + catchup=False, + tags=["managed-spark", "example"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py index 3747c9f206d47..a9961901b84f5 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_cluster_update.py @@ -83,7 +83,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "managed-spark"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_flink.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_flink.py index d852ec5c1ad96..fc95751fe451c 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_flink.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_flink.py @@ -88,7 +88,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "hadoop"], + tags=["example", "managed-spark", "hadoop"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_gke.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_gke.py index 560c4c63c484c..cabf8a517f060 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_gke.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_gke.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that show how to create a Dataproc cluster in Google Kubernetes Engine. +Example Airflow DAG that show how to create a Managed Spark cluster in Google Kubernetes Engine. Required environment variables: GKE_NAMESPACE = os.environ.get("GKE_NAMESPACE", f"{CLUSTER_NAME}") @@ -102,7 +102,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "gke"], + tags=["example", "managed-spark", "gke"], ) as dag: create_gke_cluster = GKECreateClusterOperator( task_id="create_gke_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py index a3368a7bd7579..0b20a54f190aa 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hadoop.py @@ -87,7 +87,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "hadoop"], + tags=["example", "managed-spark", "hadoop"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hive.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hive.py index 2a49be574113d..83d7ae73567ef 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hive.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_hive.py @@ -95,7 +95,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "hive"], + tags=["example", "managed-spark", "hive"], ) as dag: # [START how_to_cloud_dataproc_create_cluster_operator] create_cluster = DataprocCreateClusterOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pig.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pig.py index 712681cdea6a1..fc89a5f5e364b 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pig.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pig.py @@ -83,7 +83,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "pig"], + tags=["example", "managed-spark", "pig"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_presto.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_presto.py index 377f4aafbc82c..71ea5d653baf0 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_presto.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_presto.py @@ -87,7 +87,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "presto"], + tags=["example", "managed-spark", "presto"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py index 1024c8887077e..c6c898bbb003f 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_pyspark.py @@ -93,7 +93,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "pyspark"], + tags=["example", "managed-spark", "pyspark"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark.py index 03d963d5d1719..631a25f1064bb 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark.py @@ -83,7 +83,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "spark"], + tags=["example", "managed-spark", "spark"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py index 32bc96b03220c..ddf9554cae811 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_async.py @@ -82,7 +82,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "spark", "async"], + tags=["example", "managed-spark", "spark", "async"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py index 3f0433be20136..cb19117b7a305 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_deferrable.py @@ -84,7 +84,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "spark", "deferrable"], + tags=["example", "managed-spark", "spark", "deferrable"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py index a1a60059b4d82..26fdbe1e10262 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_spark_sql.py @@ -80,7 +80,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc"], + tags=["example", "managed-spark"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py index dc2ff3c580fde..8c4c7270b23f4 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_sparkr.py @@ -91,7 +91,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "sparkr"], + tags=["example", "managed-spark", "sparkr"], ) as dag: create_bucket = GCSCreateBucketOperator( task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_start_from_trigger.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_start_from_trigger.py index 23f1d93cd9e58..6071854f2d3a1 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_start_from_trigger.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_start_from_trigger.py @@ -81,7 +81,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "start_from_trigger"], + tags=["example", "managed-spark", "start_from_trigger"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_trino.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_trino.py index 2cc32ea52d97f..3bbbbf089f29a 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_trino.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_trino.py @@ -89,7 +89,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "trino"], + tags=["example", "managed-spark", "trino"], ) as dag: create_cluster = DataprocCreateClusterOperator( task_id="create_cluster", diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py index a0a578ec337de..7ba18427afdb4 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG for Dataproc workflow operators. +Example Airflow DAG for Managed Spark workflow operators. """ from __future__ import annotations @@ -72,7 +72,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "workflow"], + tags=["example", "managed-spark", "workflow"], ) as dag: # [START how_to_cloud_dataproc_create_workflow_template] create_workflow_template = DataprocCreateWorkflowTemplateOperator( diff --git a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py index 5a69ca4a80eea..8c34ba4bfe4ad 100644 --- a/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py +++ b/providers/google/tests/system/google/cloud/dataproc/example_dataproc_workflow_deferrable.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG for Dataproc workflow operators. +Example Airflow DAG for Managed Spark workflow operators. """ from __future__ import annotations @@ -71,7 +71,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "dataproc", "workflow", "deferrable"], + tags=["example", "managed-spark", "workflow", "deferrable"], ) as dag: create_workflow_template = DataprocCreateWorkflowTemplateOperator( task_id="create_workflow_template", diff --git a/providers/google/tests/system/google/cloud/looker/example_looker.py b/providers/google/tests/system/google/cloud/looker/example_looker.py index de746effb6c00..32287b3a32ce7 100644 --- a/providers/google/tests/system/google/cloud/looker/example_looker.py +++ b/providers/google/tests/system/google/cloud/looker/example_looker.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -Example Airflow DAG that show how to use various Looker +Example Airflow DAG that show how to use various Data Studio operators to submit PDT materialization job and manage it. """ @@ -38,7 +38,7 @@ schedule="@once", start_date=datetime(2021, 1, 1), catchup=False, - tags=["example", "looker"], + tags=["example", "data-studio"], ) as dag: # [START cloud_looker_async_start_pdt_sensor] start_pdt_task_async = LookerStartPdtBuildOperator( diff --git a/providers/google/tests/unit/google/cloud/operators/test_dataproc.py b/providers/google/tests/unit/google/cloud/operators/test_dataproc.py index 164580b6c7fbd..ecaeae1ca3bae 100644 --- a/providers/google/tests/unit/google/cloud/operators/test_dataproc.py +++ b/providers/google/tests/unit/google/cloud/operators/test_dataproc.py @@ -85,6 +85,7 @@ DATAPROC_PATH = "airflow.providers.google.cloud.operators.dataproc.{}" DATAPROC_TRIGGERS_PATH = "airflow.providers.google.cloud.triggers.dataproc.{}" + TASK_ID = "task-id" GCP_PROJECT = "test-project" GCP_REGION = "test-location" diff --git a/providers/google/tests/unit/google/cloud/operators/test_knowledge_catalog.py b/providers/google/tests/unit/google/cloud/operators/test_knowledge_catalog.py new file mode 100644 index 0000000000000..a1636554197ed --- /dev/null +++ b/providers/google/tests/unit/google/cloud/operators/test_knowledge_catalog.py @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.google.cloud.operators.dataplex import ( + DataplexCatalogCreateAspectTypeOperator, + DataplexCatalogCreateEntryGroupOperator, + DataplexCatalogCreateEntryOperator, + DataplexCatalogCreateEntryTypeOperator, + DataplexCatalogDeleteAspectTypeOperator, + DataplexCatalogDeleteEntryGroupOperator, + DataplexCatalogDeleteEntryOperator, + DataplexCatalogDeleteEntryTypeOperator, + DataplexCatalogGetAspectTypeOperator, + DataplexCatalogGetEntryGroupOperator, + DataplexCatalogGetEntryOperator, + DataplexCatalogGetEntryTypeOperator, + DataplexCatalogListAspectTypesOperator, + DataplexCatalogListEntriesOperator, + DataplexCatalogListEntryGroupsOperator, + DataplexCatalogListEntryTypesOperator, + DataplexCatalogLookupEntryOperator, + DataplexCatalogSearchEntriesOperator, + DataplexCatalogUpdateAspectTypeOperator, + DataplexCatalogUpdateEntryGroupOperator, + DataplexCatalogUpdateEntryOperator, + DataplexCatalogUpdateEntryTypeOperator, + DataplexCreateAssetOperator, + DataplexCreateLakeOperator, + DataplexCreateOrUpdateDataProfileScanOperator, + DataplexCreateOrUpdateDataQualityScanOperator, + DataplexCreateTaskOperator, + DataplexCreateZoneOperator, + DataplexDeleteAssetOperator, + DataplexDeleteDataProfileScanOperator, + DataplexDeleteDataQualityScanOperator, + DataplexDeleteLakeOperator, + DataplexDeleteTaskOperator, + DataplexDeleteZoneOperator, + DataplexGetDataProfileScanOperator, + DataplexGetDataProfileScanResultOperator, + DataplexGetDataQualityScanOperator, + DataplexGetDataQualityScanResultOperator, + DataplexGetTaskOperator, + DataplexListTasksOperator, + DataplexRunDataProfileScanOperator, + DataplexRunDataQualityScanOperator, +) +from airflow.providers.google.cloud.operators.knowledge_catalog import ( + KnowledgeCatalogCreateAspectTypeOperator, + KnowledgeCatalogCreateAssetOperator, + KnowledgeCatalogCreateEntryGroupOperator, + KnowledgeCatalogCreateEntryOperator, + KnowledgeCatalogCreateEntryTypeOperator, + KnowledgeCatalogCreateLakeOperator, + KnowledgeCatalogCreateOrUpdateDataProfileScanOperator, + KnowledgeCatalogCreateOrUpdateDataQualityScanOperator, + KnowledgeCatalogCreateTaskOperator, + KnowledgeCatalogCreateZoneOperator, + KnowledgeCatalogDeleteAspectTypeOperator, + KnowledgeCatalogDeleteAssetOperator, + KnowledgeCatalogDeleteDataProfileScanOperator, + KnowledgeCatalogDeleteDataQualityScanOperator, + KnowledgeCatalogDeleteEntryGroupOperator, + KnowledgeCatalogDeleteEntryOperator, + KnowledgeCatalogDeleteEntryTypeOperator, + KnowledgeCatalogDeleteLakeOperator, + KnowledgeCatalogDeleteTaskOperator, + KnowledgeCatalogDeleteZoneOperator, + KnowledgeCatalogGetAspectTypeOperator, + KnowledgeCatalogGetDataProfileScanOperator, + KnowledgeCatalogGetDataProfileScanResultOperator, + KnowledgeCatalogGetDataQualityScanOperator, + KnowledgeCatalogGetDataQualityScanResultOperator, + KnowledgeCatalogGetEntryGroupOperator, + KnowledgeCatalogGetEntryOperator, + KnowledgeCatalogGetEntryTypeOperator, + KnowledgeCatalogGetTaskOperator, + KnowledgeCatalogListAspectTypesOperator, + KnowledgeCatalogListEntriesOperator, + KnowledgeCatalogListEntryGroupsOperator, + KnowledgeCatalogListEntryTypesOperator, + KnowledgeCatalogListTasksOperator, + KnowledgeCatalogLookupEntryOperator, + KnowledgeCatalogRunDataProfileScanOperator, + KnowledgeCatalogRunDataQualityScanOperator, + KnowledgeCatalogSearchEntriesOperator, + KnowledgeCatalogUpdateAspectTypeOperator, + KnowledgeCatalogUpdateEntryGroupOperator, + KnowledgeCatalogUpdateEntryOperator, + KnowledgeCatalogUpdateEntryTypeOperator, +) + + +@pytest.mark.parametrize( + ("alias_operator", "original_operator"), + [ + (KnowledgeCatalogCreateTaskOperator, DataplexCreateTaskOperator), + (KnowledgeCatalogDeleteTaskOperator, DataplexDeleteTaskOperator), + (KnowledgeCatalogListTasksOperator, DataplexListTasksOperator), + (KnowledgeCatalogGetTaskOperator, DataplexGetTaskOperator), + (KnowledgeCatalogCreateLakeOperator, DataplexCreateLakeOperator), + (KnowledgeCatalogDeleteLakeOperator, DataplexDeleteLakeOperator), + ( + KnowledgeCatalogCreateOrUpdateDataQualityScanOperator, + DataplexCreateOrUpdateDataQualityScanOperator, + ), + (KnowledgeCatalogGetDataQualityScanOperator, DataplexGetDataQualityScanOperator), + (KnowledgeCatalogDeleteDataQualityScanOperator, DataplexDeleteDataQualityScanOperator), + (KnowledgeCatalogRunDataQualityScanOperator, DataplexRunDataQualityScanOperator), + ( + KnowledgeCatalogGetDataQualityScanResultOperator, + DataplexGetDataQualityScanResultOperator, + ), + ( + KnowledgeCatalogCreateOrUpdateDataProfileScanOperator, + DataplexCreateOrUpdateDataProfileScanOperator, + ), + (KnowledgeCatalogGetDataProfileScanOperator, DataplexGetDataProfileScanOperator), + (KnowledgeCatalogDeleteDataProfileScanOperator, DataplexDeleteDataProfileScanOperator), + (KnowledgeCatalogRunDataProfileScanOperator, DataplexRunDataProfileScanOperator), + ( + KnowledgeCatalogGetDataProfileScanResultOperator, + DataplexGetDataProfileScanResultOperator, + ), + (KnowledgeCatalogCreateZoneOperator, DataplexCreateZoneOperator), + (KnowledgeCatalogDeleteZoneOperator, DataplexDeleteZoneOperator), + (KnowledgeCatalogCreateAssetOperator, DataplexCreateAssetOperator), + (KnowledgeCatalogDeleteAssetOperator, DataplexDeleteAssetOperator), + (KnowledgeCatalogCreateEntryGroupOperator, DataplexCatalogCreateEntryGroupOperator), + (KnowledgeCatalogGetEntryGroupOperator, DataplexCatalogGetEntryGroupOperator), + (KnowledgeCatalogDeleteEntryGroupOperator, DataplexCatalogDeleteEntryGroupOperator), + (KnowledgeCatalogListEntryGroupsOperator, DataplexCatalogListEntryGroupsOperator), + (KnowledgeCatalogUpdateEntryGroupOperator, DataplexCatalogUpdateEntryGroupOperator), + (KnowledgeCatalogCreateEntryTypeOperator, DataplexCatalogCreateEntryTypeOperator), + (KnowledgeCatalogGetEntryTypeOperator, DataplexCatalogGetEntryTypeOperator), + (KnowledgeCatalogDeleteEntryTypeOperator, DataplexCatalogDeleteEntryTypeOperator), + (KnowledgeCatalogListEntryTypesOperator, DataplexCatalogListEntryTypesOperator), + (KnowledgeCatalogUpdateEntryTypeOperator, DataplexCatalogUpdateEntryTypeOperator), + (KnowledgeCatalogCreateAspectTypeOperator, DataplexCatalogCreateAspectTypeOperator), + (KnowledgeCatalogGetAspectTypeOperator, DataplexCatalogGetAspectTypeOperator), + (KnowledgeCatalogListAspectTypesOperator, DataplexCatalogListAspectTypesOperator), + (KnowledgeCatalogUpdateAspectTypeOperator, DataplexCatalogUpdateAspectTypeOperator), + (KnowledgeCatalogDeleteAspectTypeOperator, DataplexCatalogDeleteAspectTypeOperator), + (KnowledgeCatalogCreateEntryOperator, DataplexCatalogCreateEntryOperator), + (KnowledgeCatalogGetEntryOperator, DataplexCatalogGetEntryOperator), + (KnowledgeCatalogListEntriesOperator, DataplexCatalogListEntriesOperator), + (KnowledgeCatalogSearchEntriesOperator, DataplexCatalogSearchEntriesOperator), + (KnowledgeCatalogLookupEntryOperator, DataplexCatalogLookupEntryOperator), + (KnowledgeCatalogUpdateEntryOperator, DataplexCatalogUpdateEntryOperator), + (KnowledgeCatalogDeleteEntryOperator, DataplexCatalogDeleteEntryOperator), + ], +) +def test_knowledge_catalog_aliases(alias_operator, original_operator): + assert alias_operator is original_operator diff --git a/providers/google/tests/unit/google/cloud/operators/test_looker.py b/providers/google/tests/unit/google/cloud/operators/test_looker.py index 91f2387a64aa4..6985fd6ec77d2 100644 --- a/providers/google/tests/unit/google/cloud/operators/test_looker.py +++ b/providers/google/tests/unit/google/cloud/operators/test_looker.py @@ -23,7 +23,10 @@ from airflow.models import DAG, DagBag from airflow.providers.common.compat.sdk import AirflowException -from airflow.providers.google.cloud.operators.looker import LookerStartPdtBuildOperator +from airflow.providers.google.cloud.operators.looker import ( + DataStudioStartPdtBuildOperator, + LookerStartPdtBuildOperator, +) from airflow.utils.timezone import datetime from tests_common.test_utils.db import clear_db_runs, clear_db_xcom @@ -40,6 +43,10 @@ TEST_JOB_ID = "123" +def test_data_studio_aliases(): + assert DataStudioStartPdtBuildOperator is LookerStartPdtBuildOperator + + class LookerTestBase: @classmethod def setUpClass(cls): diff --git a/providers/google/tests/unit/google/cloud/operators/test_managed_spark.py b/providers/google/tests/unit/google/cloud/operators/test_managed_spark.py new file mode 100644 index 0000000000000..a1296125db25f --- /dev/null +++ b/providers/google/tests/unit/google/cloud/operators/test_managed_spark.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.google.cloud.operators.dataproc import ( + DataprocCreateBatchOperator, + DataprocCreateClusterOperator, + DataprocDeleteClusterOperator, + DataprocSubmitJobOperator, +) +from airflow.providers.google.cloud.operators.managed_spark import ( + ManagedSparkCreateBatchOperator, + ManagedSparkCreateClusterOperator, + ManagedSparkDeleteClusterOperator, + ManagedSparkSubmitJobOperator, +) + + +def test_managed_spark_aliases(): + assert ManagedSparkCreateClusterOperator is DataprocCreateClusterOperator + assert ManagedSparkDeleteClusterOperator is DataprocDeleteClusterOperator + assert ManagedSparkSubmitJobOperator is DataprocSubmitJobOperator + assert ManagedSparkCreateBatchOperator is DataprocCreateBatchOperator