diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-action.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-action.md index b105453bf..a72ec526f 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-action.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-action.md @@ -23,6 +23,8 @@ Properties **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that MySQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Username:** User identity for connecting to the specified database. diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsink.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsink.md index 2fa240ddf..eaf9e5535 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsink.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsink.md @@ -32,6 +32,8 @@ You also can use the macro function ${conn(connection-name)}. **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that MySQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Table Name:** Name of the table to export to. Table must exist prior to running the pipeline. diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md index 1c656ad51..52a5945e7 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md @@ -31,6 +31,8 @@ You also can use the macro function ${conn(connection-name)}. **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that MySQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Import Query:** The SELECT query to use to import data from the specified table. diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md index 7ddcaedbb..3197760e0 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md @@ -18,6 +18,8 @@ Properties **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that MySQL is running on. + **Database:** MySQL database name. **Username:** User identity for connecting to the specified database. Required for databases that need diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java index 551102118..0608edb75 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java @@ -18,11 +18,13 @@ import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.action.AbstractDBAction; import io.cdap.plugin.db.action.QueryConfig; import io.cdap.plugin.util.CloudSQLUtil; @@ -48,11 +50,13 @@ public CloudSQLMySQLAction(CloudSQLMySQLActionConfig cloudsqlMysqlActionConfig) @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - - CloudSQLUtil.checkConnectionName( - failureCollector, - cloudsqlMysqlActionConfig.instanceType, - cloudsqlMysqlActionConfig.connectionName); + + if (cloudsqlMysqlActionConfig.canConnect()) { + CloudSQLUtil.checkConnectionName( + failureCollector, + cloudsqlMysqlActionConfig.instanceType, + cloudsqlMysqlActionConfig.connectionName); + } super.configurePipeline(pipelineConfigurer); } @@ -69,10 +73,18 @@ public CloudSQLMySQLActionConfig() { "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " + "instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.") + @Macro public String connectionName; + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + @Name(DATABASE) @Description("Database name to connect to") + @Macro public String database; @Name(CloudSQLMySQLConstants.CONNECTION_TIMEOUT) @@ -94,6 +106,7 @@ public String getConnectionString() { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, connectionName, + getPort(), database); } @@ -103,10 +116,19 @@ public String getConnectionString() { connectionName); } + public int getPort() { + return port == null ? 3306 : port; + } + @Override public Map getDBSpecificArguments() { return ImmutableMap.of( CloudSQLMySQLConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + + public boolean canConnect() { + return !containsMacro(CloudSQLUtil.CONNECTION_NAME) && !containsMacro(ConnectionConfig.PORT) && + !containsMacro(DATABASE); + } } } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java index 42b3227ab..1e89d5a95 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java @@ -17,6 +17,7 @@ package io.cdap.plugin.cloudsql.mysql; import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; @@ -38,10 +39,18 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig { "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " + "instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.") + @Macro private String connectionName; + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + @Name(ConnectionConfig.DATABASE) @Description("Database name to connect to") + @Macro private String database; @Name(CloudSQLUtil.INSTANCE_TYPE) @@ -49,7 +58,8 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig { private String instanceType; public CloudSQLMySQLConnectorConfig(String user, String password, String jdbcPluginName, String connectionArguments, - String instanceType, String connectionName, String database) { + String instanceType, String connectionName, String database, + @Nullable Integer port) { this.user = user; this.password = password; this.jdbcPluginName = jdbcPluginName; @@ -57,6 +67,7 @@ public CloudSQLMySQLConnectorConfig(String user, String password, String jdbcPlu this.instanceType = instanceType; this.connectionName = connectionName; this.database = database; + this.port = port; } public String getDatabase() { @@ -71,12 +82,17 @@ public String getConnectionName() { return connectionName; } + public int getPort() { + return port == null ? 3306 : port; + } + @Override public String getConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, connectionName, + getPort(), database); } @@ -93,4 +109,10 @@ public Properties getConnectionArgumentsProperties() { properties.put(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); return properties; } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(CloudSQLUtil.CONNECTION_NAME) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java index ae8a34c6a..c4b0d3b0f 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConstants.java @@ -26,5 +26,5 @@ private CloudSQLMySQLConstants() { public static final String CONNECTION_TIMEOUT = "connectionTimeout"; public static final String PUBLIC_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory"; - public static final String PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s/%s"; + public static final String PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s:%s/%s"; } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index ad7f63b1e..271012f7e 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -91,7 +91,8 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { host = connectionParams[2]; location = connectionParams[1]; } - String fqn = DBUtils.constructFQN("mysql", host, 3306, + String fqn = DBUtils.constructFQN("mysql", host, + cloudsqlMysqlSinkConfig.getConnection().getPort(), cloudsqlMysqlSinkConfig.getConnection().getDatabase(), cloudsqlMysqlSinkConfig.getReferenceName()); Asset.Builder assetBuilder = Asset.builder(cloudsqlMysqlSinkConfig.getReferenceName()).setFqn(fqn); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 50e2c3b7a..b8b6fbf27 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -86,6 +86,7 @@ protected String createConnectionString() { return String.format( CloudSQLMySQLConstants.PRIVATE_CLOUDSQL_MYSQL_CONNECTION_STRING_FORMAT, cloudsqlMysqlSourceConfig.connection.getConnectionName(), + cloudsqlMysqlSourceConfig.connection.getPort(), cloudsqlMysqlSourceConfig.connection.getDatabase()); } @@ -108,7 +109,8 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { host = connectionParams[2]; location = connectionParams[1]; } - String fqn = DBUtils.constructFQN("mysql", host, 3306, + String fqn = DBUtils.constructFQN("mysql", host, + cloudsqlMysqlSourceConfig.getConnection().getPort(), cloudsqlMysqlSourceConfig.getConnection().getDatabase(), cloudsqlMysqlSourceConfig.getReferenceName()); Asset.Builder assetBuilder = Asset.builder(cloudsqlMysqlSourceConfig.getReferenceName()).setFqn(fqn); diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorTest.java index f8fc20b70..610b138dd 100644 --- a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorTest.java +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorTest.java @@ -54,7 +54,7 @@ public void test() throws IOException, ClassNotFoundException, InstantiationExce test( new CloudSQLMySQLConnector( new CloudSQLMySQLConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, instanceType, - connectionName, database) + connectionName, database, null) ), JDBC_DRIVER_CLASS_NAME, CloudSQLMySQLConstants.PLUGIN_NAME diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json index 42405fcce..66d6ebb85 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json @@ -74,11 +74,11 @@ } }, { - "widget-type": "textbox", - "label": "Instance Name", - "name": "instanceName", + "widget-type": "number", + "label": "Port", + "name": "port", "widget-attributes": { - "placeholder": "CloudSQL instance connection name" + "default": "3306" } }, { @@ -113,5 +113,19 @@ } ] } + ], + "filters": [ + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] + } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json index 5720b3d35..89a7d7736 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json @@ -66,6 +66,14 @@ "placeholder": "CloudSQL instance connection name" } }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "3306" + } + }, { "widget-type": "textbox", "label": "Username", @@ -214,6 +222,18 @@ "name": "connection" } ] + }, + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] } ], "outputs": [], diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json index ef75a8a29..4ac7747f4 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json @@ -66,6 +66,14 @@ "placeholder": "CloudSQL instance connection name" } }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "3306" + } + }, { "widget-type": "textbox", "label": "Username", @@ -231,6 +239,18 @@ "name": "connection" } ] + }, + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] } ], "jump-config": { diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index 1f8551605..b5c2c9993 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -46,6 +46,14 @@ "widget-attributes": { "placeholder": "CloudSQL instance connection name" } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "3306" + } } ] }, @@ -88,5 +96,19 @@ ] } ], - "outputs": [] + "outputs": [], + "filters": [ + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] + } + ] } diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-action.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-action.md index 7ad2f51b4..c8aefd58a 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-action.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-action.md @@ -23,6 +23,8 @@ Properties **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that PostgreSQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Username:** User identity for connecting to the specified database. diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md index 079d5df32..4f2107e96 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsink.md @@ -32,6 +32,8 @@ You also can use the macro function ${conn(connection-name)}. **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that PostgreSQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Table Name:** Name of the table to export to. diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md index 3c3bd989e..c13fb06fe 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md @@ -31,6 +31,8 @@ You also can use the macro function ${conn(connection-name)}. **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that PostgreSQL is running on. + **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. **Import Query:** The SELECT query to use to import data from the specified table. diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md index 4bee117af..0e502fefd 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md @@ -18,6 +18,8 @@ Properties **Connection Name:** The CloudSQL instance to connect to in the format :\:. Can be found in the instance overview page. +**Port:** Port that PostgreSQL is running on. + **Database:** CloudSQL PostgreSQL database name. **Username:** User identity for connecting to the specified database. Required for databases that need diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java index 072ff2c8f..1a3f8ad7b 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java @@ -18,11 +18,13 @@ import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.action.AbstractDBAction; import io.cdap.plugin.db.action.QueryConfig; import io.cdap.plugin.util.CloudSQLUtil; @@ -48,11 +50,13 @@ public CloudSQLPostgreSQLAction(CloudSQLPostgreSQLActionConfig cloudsqlPostgresq @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); - - CloudSQLUtil.checkConnectionName( - failureCollector, - cloudsqlPostgresqlActionConfig.instanceType, - cloudsqlPostgresqlActionConfig.connectionName); + + if (cloudsqlPostgresqlActionConfig.canConnect()) { + CloudSQLUtil.checkConnectionName( + failureCollector, + cloudsqlPostgresqlActionConfig.instanceType, + cloudsqlPostgresqlActionConfig.connectionName); + } super.configurePipeline(pipelineConfigurer); } @@ -69,10 +73,18 @@ public CloudSQLPostgreSQLActionConfig() { "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " + "instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.") + @Macro public String connectionName; + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + @Name(DATABASE) @Description("Database name to connect to") + @Macro public String database; @Name(CloudSQLPostgreSQLConstants.CONNECTION_TIMEOUT) @@ -94,6 +106,7 @@ public String getConnectionString() { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, connectionName, + getPort(), database); } @@ -103,10 +116,19 @@ public String getConnectionString() { connectionName); } + public int getPort() { + return port == null ? 5432 : port; + } + @Override public Map getDBSpecificArguments() { return ImmutableMap.of( CloudSQLPostgreSQLConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + + public boolean canConnect() { + return !containsMacro(CloudSQLUtil.CONNECTION_NAME) && !containsMacro(ConnectionConfig.PORT) && + !containsMacro(DATABASE); + } } } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java index d3ec0849b..30effd350 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java @@ -17,6 +17,7 @@ package io.cdap.plugin.cloudsql.postgres; import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; @@ -34,10 +35,18 @@ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig "The CloudSQL instance to connect to. For a public instance, the connection string should be in the format " + ":: which can be found in the instance overview page. For a private " + "instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.") + @Macro private String connectionName; + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + @Name(ConnectionConfig.DATABASE) @Description("Database name to connect to") + @Macro private String database; @Name(CloudSQLUtil.INSTANCE_TYPE) @@ -46,7 +55,7 @@ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig public CloudSQLPostgreSQLConnectorConfig(String username, String password, String jdbcPluginName, String connectionArguments, String instanceType, - String connectionName, String database) { + String connectionName, String database, @Nullable Integer port) { this.user = username; this.password = password; this.jdbcPluginName = jdbcPluginName; @@ -54,6 +63,7 @@ public CloudSQLPostgreSQLConnectorConfig(String username, String password, Strin this.instanceType = instanceType; this.connectionName = connectionName; this.database = database; + this.port = port; } public String getDatabase() { @@ -68,12 +78,17 @@ public String getConnectionName() { return connectionName; } + public int getPort() { + return port == null ? 5432 : port; + } + @Override public String getConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase(instanceType)) { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, connectionName, + getPort(), database); } @@ -82,4 +97,10 @@ public String getConnectionString() { database, connectionName); } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(CloudSQLUtil.CONNECTION_NAME) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java index 946171102..8296ed344 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConstants.java @@ -26,5 +26,5 @@ private CloudSQLPostgreSQLConstants() { public static final String CONNECTION_TIMEOUT = "connectionTimeout"; public static final String PUBLIC_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.postgres.SocketFactory"; - public static final String PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql://%s/%s"; + public static final String PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT = "jdbc:postgresql://%s:%s/%s"; } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index d74358f54..fa5d66f5e 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -136,7 +136,8 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { host = connectionParams[2]; location = connectionParams[1]; } - String fqn = DBUtils.constructFQN("postgres", host, 5432, + String fqn = DBUtils.constructFQN("postgres", host, + cloudsqlPostgresqlSinkConfig.getConnection().getPort(), cloudsqlPostgresqlSinkConfig.getConnection().getDatabase(), cloudsqlPostgresqlSinkConfig.getReferenceName()); Asset.Builder assetBuilder = Asset.builder(cloudsqlPostgresqlSinkConfig.getReferenceName()).setFqn(fqn); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 6eb1765da..6d6ba29f8 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -93,6 +93,7 @@ protected String createConnectionString() { return String.format( CloudSQLPostgreSQLConstants.PRIVATE_CLOUDSQL_POSTGRES_CONNECTION_STRING_FORMAT, cloudsqlPostgresqlSourceConfig.connection.getConnectionName(), + cloudsqlPostgresqlSourceConfig.connection.getPort(), cloudsqlPostgresqlSourceConfig.connection.getDatabase()); } @@ -116,7 +117,8 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { host = connectionParams[2]; location = connectionParams[1]; } - String fqn = DBUtils.constructFQN("postgres", host, 5432, + String fqn = DBUtils.constructFQN("postgres", host, + cloudsqlPostgresqlSourceConfig.getConnection().getPort(), cloudsqlPostgresqlSourceConfig.getConnection().getDatabase(), cloudsqlPostgresqlSourceConfig.getReferenceName()); Asset.Builder assetBuilder = Asset.builder(cloudsqlPostgresqlSourceConfig.getReferenceName()).setFqn(fqn); diff --git a/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorTest.java b/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorTest.java index dce4ed443..215417114 100644 --- a/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorTest.java +++ b/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorTest.java @@ -54,7 +54,7 @@ public void test() throws IOException, ClassNotFoundException, InstantiationExce test( new CloudSQLPostgreSQLConnector( new CloudSQLPostgreSQLConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, instanceType, - connectionName, database) + connectionName, database, null) ), JDBC_DRIVER_CLASS_NAME, CloudSQLPostgreSQLConstants.PLUGIN_NAME diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json index 9e9f124b9..eab240679 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json @@ -73,6 +73,14 @@ "placeholder": "CloudSQL instance connection name" } }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5432" + } + }, { "widget-type": "textarea", "label": "Database Command", @@ -105,5 +113,19 @@ } ] } + ], + "filters": [ + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] + } ] } diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 764106d09..2fda594dd 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -66,6 +66,14 @@ "placeholder": "CloudSQL instance connection name" } }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5432" + } + }, { "widget-type": "textbox", "label": "Username", @@ -235,6 +243,18 @@ "name": "connection" } ] + }, + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] } ], "jump-config": { diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 825d5714c..96ea97ac2 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -66,6 +66,14 @@ "placeholder": "CloudSQL instance connection name" } }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5432" + } + }, { "widget-type": "textbox", "label": "Username", @@ -235,6 +243,18 @@ "name": "connection" } ] + }, + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] } ], "jump-config": { diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index de3af0795..9824f91bd 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -46,6 +46,14 @@ "widget-attributes": { "placeholder": "CloudSQL instance connection name" } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5432" + } } ] }, @@ -88,5 +96,19 @@ ] } ], - "outputs": [] + "outputs": [], + "filters": [ + { + "name": "showPrivateInstanceProperties ", + "condition": { + "expression": "instanceType == 'private'" + }, + "show": [ + { + "type": "property", + "name": "port" + } + ] + } + ] }