OPERATOR-240 Unable to set image urls when using operator in air gapped environment

drivers/storage/portworx/component/autopilot.go

@@ -307,7 +307,7 @@ func (c *autopilot) createDeployment(
 	sort.Strings(argList)
 	command := append([]string{"/autopilot"}, argList...)
 
-	imageName = util.GetImageURN(cluster.Spec.CustomImageRegistry, imageName)
+	imageName = util.GetImageURN(cluster, imageName)
 
 	envMap := make(map[string]*v1.EnvVar)
 	envMap[pxutil.EnvKeyPortworxNamespace] = &v1.EnvVar{

drivers/storage/portworx/component/csi.go

@@ -403,24 +403,24 @@ func (c *csi) createDeployment(
 	)
 
 	provisionerImage = util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.CSIProvisioner,
 	)
 	if csiConfig.IncludeAttacher && cluster.Status.DesiredImages.CSIAttacher != "" {
 		attacherImage = util.GetImageURN(
-			cluster.Spec.CustomImageRegistry,
+			cluster,
 			cluster.Status.DesiredImages.CSIAttacher,
 		)
 	}
 	if csiConfig.IncludeSnapshotter && cluster.Status.DesiredImages.CSISnapshotter != "" {
 		snapshotterImage = util.GetImageURN(
-			cluster.Spec.CustomImageRegistry,
+			cluster,
 			cluster.Status.DesiredImages.CSISnapshotter,
 		)
 	}
 	if csiConfig.IncludeResizer && cluster.Status.DesiredImages.CSIResizer != "" {
 		resizerImage = util.GetImageURN(
-			cluster.Spec.CustomImageRegistry,
+			cluster,
 			cluster.Status.DesiredImages.CSIResizer,
 		)
 	}
@@ -695,11 +695,11 @@ func (c *csi) createStatefulSet(
 	)
 
 	provisionerImage = util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.CSIProvisioner,
 	)
 	attacherImage = util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.CSIAttacher,
 	)
 

drivers/storage/portworx/component/lighthouse.go

@@ -328,10 +328,9 @@ func (c *lighthouse) createDeployment(
 		}
 	}
 
-	imageRegistry := cluster.Spec.CustomImageRegistry
-	lhImage = util.GetImageURN(imageRegistry, lhImage)
-	configSyncImage = util.GetImageURN(imageRegistry, configSyncImage)
-	storkConnectorImage = util.GetImageURN(imageRegistry, storkConnectorImage)
+	lhImage = util.GetImageURN(cluster, lhImage)
+	configSyncImage = util.GetImageURN(cluster, configSyncImage)
+	storkConnectorImage = util.GetImageURN(cluster, storkConnectorImage)
 
 	modified := lhImage != existingLhImage ||
 		configSyncImage != existingConfigInitImage ||

drivers/storage/portworx/component/portworx_api.go

@@ -158,7 +158,7 @@ func (c *portworxAPI) createDaemonSet(
 		existingImageName = existingDaemonSet.Spec.Template.Spec.Containers[0].Image
 	}
 
-	imageName := util.GetImageURN(cluster.Spec.CustomImageRegistry, pxutil.ImageNamePause)
+	imageName := util.GetImageURN(cluster, pxutil.ImageNamePause)
 	serviceAccount := pxutil.PortworxServiceAccountName(cluster)
 	existingServiceAccount := existingDaemonSet.Spec.Template.Spec.ServiceAccountName
 

drivers/storage/portworx/component/portworx_proxy.go

@@ -214,7 +214,7 @@ func (c *portworxProxy) createDaemonSet(
 		existingImageName = existingDaemonSet.Spec.Template.Spec.Containers[0].Image
 	}
 
-	imageName := util.GetImageURN(cluster.Spec.CustomImageRegistry, "k8s.gcr.io/pause:3.1")
+	imageName := util.GetImageURN(cluster, "k8s.gcr.io/pause:3.1")
 
 	modified := existingImageName != imageName ||
 		util.HasPullSecretChanged(cluster, existingDaemonSet.Spec.Template.Spec.ImagePullSecrets) ||

drivers/storage/portworx/component/prometheus.go

@@ -390,7 +390,7 @@ func (c *prometheus) createOperatorDeployment(
 	}
 
 	imageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.PrometheusOperator,
 	)
 
@@ -421,11 +421,11 @@ func getPrometheusOperatorDeploymentSpec(
 		"k8s-app": PrometheusOperatorDeploymentName,
 	}
 	configReloaderImageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.PrometheusConfigMapReload,
 	)
 	prometheusConfigReloaderImageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.PrometheusConfigReloader,
 	)
 	args := make([]string, 0)
@@ -540,7 +540,7 @@ func (c *prometheus) createPrometheusInstance(
 ) error {
 	replicas := int32(1)
 	prometheusImageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		cluster.Status.DesiredImages.Prometheus,
 	)
 

drivers/storage/portworx/component/pvccontroller.go

@@ -275,7 +275,7 @@ func (c *pvcController) createDeployment(
 		kubeControllerImage = "k8s.gcr.io/kube-controller-manager-amd64"
 	}
 	imageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		kubeControllerImage+":v"+c.k8sVersion.String(),
 	)
 

drivers/storage/portworx/deployment.go

@@ -427,7 +427,7 @@ func configureStorageNodeSpec(node *corev1.StorageNode, config *cloudstorage.Con
 }
 
 func (t *template) portworxContainer() v1.Container {
-	pxImage := util.GetImageURN(t.cluster.Spec.CustomImageRegistry, t.cluster.Spec.Image)
+	pxImage := util.GetImageURN(t.cluster, t.cluster.Spec.Image)
 	return v1.Container{
 		Name:            pxContainerName,
 		Image:           pxImage,
@@ -464,7 +464,7 @@ func (t *template) portworxContainer() v1.Container {
 }
 
 func (t *template) kvdbContainer() v1.Container {
-	kvdbProxyImage := util.GetImageURN(t.cluster.Spec.CustomImageRegistry, pxutil.ImageNamePause)
+	kvdbProxyImage := util.GetImageURN(t.cluster, pxutil.ImageNamePause)
 	kvdbTargetPort := 9019
 	if t.startPort != pxutil.DefaultStartPort {
 		kvdbTargetPort = t.startPort + 15
@@ -530,7 +530,7 @@ func (t *template) csiRegistrarContainer() *v1.Container {
 	if t.cluster.Status.DesiredImages.CSINodeDriverRegistrar != "" {
 		container.Name = "csi-node-driver-registrar"
 		container.Image = util.GetImageURN(
-			t.cluster.Spec.CustomImageRegistry,
+			t.cluster,
 			t.cluster.Status.DesiredImages.CSINodeDriverRegistrar,
 		)
 		container.Args = []string{
@@ -541,7 +541,7 @@ func (t *template) csiRegistrarContainer() *v1.Container {
 	} else if t.cluster.Status.DesiredImages.CSIDriverRegistrar != "" {
 		container.Name = "csi-driver-registrar"
 		container.Image = util.GetImageURN(
-			t.cluster.Spec.CustomImageRegistry,
+			t.cluster,
 			t.cluster.Status.DesiredImages.CSIDriverRegistrar,
 		)
 		container.Args = []string{
@@ -562,7 +562,7 @@ func (t *template) telemetryContainer() *v1.Container {
 	container := v1.Container{
 		Name: "telemetry",
 		Image: util.GetImageURN(
-			t.cluster.Spec.CustomImageRegistry,
+			t.cluster,
 			t.getDesiredTelemetryImage(t.cluster),
 		),
 		ImagePullPolicy: t.imagePullPolicy,

drivers/storage/portworx/uninstall.go

@@ -180,7 +180,7 @@ func (u *uninstallPortworx) RunNodeWiper(
 		release := manifest.Instance().GetVersions(u.cluster, true)
 		wiperImage = release.Components.NodeWiper
 	}
-	wiperImage = util.GetImageURN(u.cluster.Spec.CustomImageRegistry, wiperImage)
+	wiperImage = util.GetImageURN(u.cluster, wiperImage)
 
 	args := []string{"-w"}
 	if removeData {

pkg/constants/metadata.go

@@ -28,6 +28,12 @@ const (
 	// AnnotationPodSafeToEvict annotation tells cluster autoscaler whether the
 	// pod is safe to be evicted when scaling down a node
 	AnnotationPodSafeToEvict = "cluster-autoscaler.kubernetes.io/safe-to-evict"
+
+	// AnnotationCommonImageRegistries annotation contains the common image registries, separated by comma.
+	// When custom image registry is provided, we will replace any image with common registry with
+	// the custom registry, there is a list of hardcoded common registries, however the list
+	// may not be complete, users can use this annotation to add more.
+	AnnotationCommonImageRegistries = OperatorPrefix + "/common-image-registries"
 )
 
 const (

pkg/controller/storagecluster/stork.go

@@ -548,7 +548,7 @@ func (c *Controller) createStorkDeployment(
 	sort.Strings(argList)
 	command := append([]string{"/stork"}, argList...)
 
-	imageName = util.GetImageURN(cluster.Spec.CustomImageRegistry, imageName)
+	imageName = util.GetImageURN(cluster, imageName)
 	hostNetwork := cluster.Spec.Stork.HostNetwork != nil && *cluster.Spec.Stork.HostNetwork
 
 	envMap := c.Driver.GetStorkEnvMap(cluster)
@@ -764,7 +764,7 @@ func (c *Controller) createStorkSchedDeployment(
 		kubeSchedImage = "k8s.gcr.io/kube-scheduler-amd64"
 	}
 	imageName := util.GetImageURN(
-		cluster.Spec.CustomImageRegistry,
+		cluster,
 		kubeSchedImage+":v"+c.kubernetesVersion.String(),
 	)
 

pkg/util/test/util.go

@@ -686,7 +686,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl
 			storkImageName = cluster.Spec.Stork.Image
 		}
 
-		storkImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, storkImageName)
+		storkImage := util.GetImageURN(cluster, storkImageName)
 		err := validateImageOnPods(storkImage, cluster.Namespace, map[string]string{"name": "stork"})
 		if err != nil {
 			return err
@@ -723,7 +723,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl
 			autopilotImageName = cluster.Spec.Autopilot.Image
 		}
 
-		autopilotImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, autopilotImageName)
+		autopilotImage := util.GetImageURN(cluster, autopilotImageName)
 		if err = validateImageOnPods(autopilotImage, cluster.Namespace, map[string]string{"name": "autopilot"}); err != nil {
 			return err
 		}
@@ -748,7 +748,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl
 			lighthouseImageName = cluster.Spec.UserInterface.Image
 		}
 
-		lhImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, lighthouseImageName)
+		lhImage := util.GetImageURN(cluster, lighthouseImageName)
 		if err = validateImageOnPods(lhImage, cluster.Namespace, map[string]string{"name": "lighthouse"}); err != nil {
 			return err
 		}

pkg/util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"fmt"
+	"github.com/libopenstorage/operator/pkg/constants"
 	"path"
 	"reflect"
 	"strings"
@@ -39,12 +40,35 @@ var (
 	}
 )
 
+func getMergedCommonRegistries(cluster *corev1.StorageCluster) map[string]bool {
+	val, ok := cluster.Annotations[constants.AnnotationCommonImageRegistries]
+
+	if !ok {
+		return commonDockerRegistries
+	}
+
+	mergedCommonRegistries := make(map[string]bool)
+
+	for _, v := range strings.Split(strings.TrimSpace(val), ",") {
+		mergedCommonRegistries[v] = true
+	}
+
+	for k, v := range commonDockerRegistries {
+		mergedCommonRegistries[k] = v
+	}
+
+	return mergedCommonRegistries
+}
+
 // GetImageURN returns the complete image name based on the registry and repo
-func GetImageURN(registryAndRepo, image string) string {
+func GetImageURN(cluster *corev1.StorageCluster, image string) string {
 	if image == "" {
 		return ""
 	}
 
+	registryAndRepo := cluster.Spec.CustomImageRegistry
+	mergedCommonRegistries := getMergedCommonRegistries(cluster)
+
 	omitRepo := false
 	if strings.HasSuffix(registryAndRepo, "//") {
 		omitRepo = true
@@ -59,7 +83,7 @@ func GetImageURN(registryAndRepo, image string) string {
 	imgParts := strings.Split(image, "/")
 	if len(imgParts) > 1 {
 		// advance imgParts to swallow the common registry
-		if _, present := commonDockerRegistries[imgParts[0]]; present {
+		if _, present := mergedCommonRegistries[imgParts[0]]; present {
 			imgParts = imgParts[1:]
 		}
 	}