Auto failover3

sdk/spring/ci.yml

@@ -547,26 +547,26 @@ extends:
       - name: spring-cloud-azure-appconfiguration-config
         groupId: com.azure.spring
         safeName: springcloudazureappconfigurationconfig
-        skipPublishDocGithubIo: true
-        skipPublishDocMs: true
+        skipPublishDocGithubIo: false
+        skipPublishDocMs: false
         releaseInBatch: ${{ parameters.release_springcloudazureappconfigurationconfig }}
       - name: spring-cloud-azure-appconfiguration-config-web
         groupId: com.azure.spring
         safeName: springcloudazureappconfigurationconfigweb
-        skipPublishDocGithubIo: true
-        skipPublishDocMs: true
+        skipPublishDocGithubIo: false
+        skipPublishDocMs: false
         releaseInBatch: ${{ parameters.release_springcloudazureappconfigurationconfigweb }}
       - name: spring-cloud-azure-feature-management
         groupId: com.azure.spring
         safeName: springcloudazurefeaturemanagement
-        skipPublishDocGithubIo: true
-        skipPublishDocMs: true
+        skipPublishDocGithubIo: false
+        skipPublishDocMs: false
         releaseInBatch: ${{ parameters.release_springcloudazurefeaturemanagement }}
       - name: spring-cloud-azure-feature-management-web
         groupId: com.azure.spring
         safeName: springcloudazurefeaturemanagementweb
-        skipPublishDocGithubIo: true
-        skipPublishDocMs: true
+        skipPublishDocGithubIo: false
+        skipPublishDocMs: false
         releaseInBatch: ${{ parameters.release_springcloudazurefeaturemanagementweb }}
       - name: spring-cloud-azure-starter-appconfiguration-config
         groupId: com.azure.spring

sdk/spring/spring-cloud-azure-appconfiguration-config/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features Added
 
+* Added Auto fail over support. Will automatically find Azure App Configuration replica stores for provided store. The found replica stores will be used as fail over stores after all provided replicas have failed.
+* Added property to disable auto fail over support `spring.cloud.azure.appconfiguration.stores[0].replica-discovery-enabled`.
+
 ### Breaking Changes
 
 ### Bugs Fixed

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/AppConfigurationAutoConfiguration.java

@@ -12,6 +12,7 @@
 
 import com.azure.spring.cloud.appconfiguration.config.implementation.AppConfigurationPullRefresh;
 import com.azure.spring.cloud.appconfiguration.config.implementation.AppConfigurationReplicaClientFactory;
+import com.azure.spring.cloud.appconfiguration.config.implementation.autofailover.ReplicaLookUp;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationProperties;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationProviderProperties;
 
@@ -45,9 +46,9 @@ public AppConfigurationWatchAutoConfiguration() {
         @Bean
         @ConditionalOnMissingBean
         AppConfigurationRefresh appConfigurationRefresh(AppConfigurationProperties properties,
-            AppConfigurationProviderProperties appProperties, AppConfigurationReplicaClientFactory clientFactory) {
+            AppConfigurationProviderProperties appProperties, AppConfigurationReplicaClientFactory clientFactory, ReplicaLookUp replicaLookUp) {
             return new AppConfigurationPullRefresh(clientFactory, properties.getRefreshInterval(),
-                appProperties.getDefaultMinBackoff());
+                appProperties.getDefaultMinBackoff(), replicaLookUp);
         }
     }
 }

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/implementation/AppConfigurationPropertySourceLocator.java

@@ -22,6 +22,7 @@
 import org.springframework.util.StringUtils;
 
 import com.azure.data.appconfiguration.models.ConfigurationSetting;
+import com.azure.spring.cloud.appconfiguration.config.implementation.autofailover.ReplicaLookUp;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationKeyValueSelector;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationProviderProperties;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationStoreMonitoring;
@@ -47,6 +48,8 @@ public final class AppConfigurationPropertySourceLocator implements PropertySour
     private final AppConfigurationReplicaClientFactory clientFactory;
 
     private final AppConfigurationKeyVaultClientFactory keyVaultClientFactory;
+
+    private final ReplicaLookUp replicaLookUp;
 
     private Duration refreshInterval;
 
@@ -62,12 +65,13 @@ public final class AppConfigurationPropertySourceLocator implements PropertySour
      */
     public AppConfigurationPropertySourceLocator(AppConfigurationProviderProperties appProperties,
         AppConfigurationReplicaClientFactory clientFactory, AppConfigurationKeyVaultClientFactory keyVaultClientFactory,
-        Duration refreshInterval, List<ConfigStore> configStores) {
+        Duration refreshInterval, List<ConfigStore> configStores, ReplicaLookUp replicaLookUp) {
         this.refreshInterval = refreshInterval;
         this.appProperties = appProperties;
         this.configStores = configStores;
         this.clientFactory = clientFactory;
         this.keyVaultClientFactory = keyVaultClientFactory;
+        this.replicaLookUp = replicaLookUp;
 
         BackoffTimeCalculator.setDefaults(appProperties.getDefaultMaxBackoff(), appProperties.getDefaultMinBackoff());
     }
@@ -77,6 +81,7 @@ public PropertySource<?> locate(Environment environment) {
         if (!(environment instanceof ConfigurableEnvironment)) {
             return null;
         }
+        replicaLookUp.updateAutoFailoverEndpoints();
 
         ConfigurableEnvironment env = (ConfigurableEnvironment) environment;
         boolean currentlyLoaded = env.getPropertySources().stream().anyMatch(source -> {

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/implementation/AppConfigurationPullRefresh.java

@@ -19,6 +19,7 @@
 import com.azure.spring.cloud.appconfiguration.config.AppConfigurationRefresh;
 import com.azure.spring.cloud.appconfiguration.config.AppConfigurationStoreHealth;
 import com.azure.spring.cloud.appconfiguration.config.implementation.AppConfigurationRefreshUtil.RefreshEventData;
+import com.azure.spring.cloud.appconfiguration.config.implementation.autofailover.ReplicaLookUp;
 import com.azure.spring.cloud.appconfiguration.config.implementation.http.policy.BaseAppConfigurationPolicy;
 
 import reactor.core.publisher.Mono;
@@ -42,6 +43,8 @@ public class AppConfigurationPullRefresh implements AppConfigurationRefresh, Env
     private final Duration refreshInterval;
 
     private List<String> profiles;
+
+    private final ReplicaLookUp replicaLookUp;
 
     /**
      * Component used for checking for and triggering configuration refreshes.
@@ -51,10 +54,11 @@ public class AppConfigurationPullRefresh implements AppConfigurationRefresh, Env
      * @param defaultMinBackoff minimum time between backoff retries minimum backoff time
      */
     public AppConfigurationPullRefresh(AppConfigurationReplicaClientFactory clientFactory, Duration refreshInterval,
-        Long defaultMinBackoff) {
+        Long defaultMinBackoff, ReplicaLookUp replicaLookUp) {
         this.defaultMinBackoff = defaultMinBackoff;
         this.refreshInterval = refreshInterval;
         this.clientFactory = clientFactory;
+        this.replicaLookUp = replicaLookUp;
 
     }
 
@@ -101,7 +105,7 @@ private boolean refreshStores() {
             BaseAppConfigurationPolicy.setWatchRequests(true);
             try {
                 RefreshEventData eventData = AppConfigurationRefreshUtil.refreshStoresCheck(clientFactory,
-                    refreshInterval, profiles, defaultMinBackoff);
+                    refreshInterval, profiles, defaultMinBackoff, replicaLookUp);
                 if (eventData.getDoRefresh()) {
                     publisher.publishEvent(new RefreshEvent(this, eventData, eventData.getMessage()));
                     return true;

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/implementation/AppConfigurationRefreshUtil.java

@@ -15,9 +15,11 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.util.StringUtils;
 
+import com.azure.core.exception.HttpResponseException;
 import com.azure.data.appconfiguration.models.ConfigurationSetting;
 import com.azure.data.appconfiguration.models.FeatureFlagConfigurationSetting;
 import com.azure.data.appconfiguration.models.SettingSelector;
+import com.azure.spring.cloud.appconfiguration.config.implementation.autofailover.ReplicaLookUp;
 import com.azure.spring.cloud.appconfiguration.config.implementation.http.policy.BaseAppConfigurationPolicy;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.AppConfigurationStoreMonitoring;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.FeatureFlagKeyValueSelector;
@@ -34,7 +36,7 @@ class AppConfigurationRefreshUtil {
      * @return If a refresh event is called.
      */
     static RefreshEventData refreshStoresCheck(AppConfigurationReplicaClientFactory clientFactory,
-        Duration refreshInterval, List<String> profiles, Long defaultMinBackoff) {
+        Duration refreshInterval, List<String> profiles, Long defaultMinBackoff, ReplicaLookUp replicaLookUp) {
         RefreshEventData eventData = new RefreshEventData();
         BaseAppConfigurationPolicy.setWatchRequests(true);
 
@@ -62,14 +64,14 @@ static RefreshEventData refreshStoresCheck(AppConfigurationReplicaClientFactory
                 if (monitor.isEnabled() && StateHolder.getLoadState(originEndpoint)) {
                     for (AppConfigurationReplicaClient client : clients) {
                         try {
-                            refreshWithTime(client, StateHolder.getState(originEndpoint), monitor.getRefreshInterval(), eventData);
+                            refreshWithTime(client, StateHolder.getState(originEndpoint), monitor.getRefreshInterval(), eventData, replicaLookUp);
                             if (eventData.getDoRefresh()) {
                                 clientFactory.setCurrentConfigStoreClient(originEndpoint, client.getEndpoint());
                                 return eventData;
                             }
                             // If check didn't throw an error other clients don't need to be checked.
                             break;
-                        } catch (AppConfigurationStatusException e) {
+                        } catch (HttpResponseException e) {
                             LOGGER.warn(
                                 "Failed attempting to connect to " + client.getEndpoint() + " during refresh check.");
 
@@ -87,14 +89,14 @@ static RefreshEventData refreshStoresCheck(AppConfigurationReplicaClientFactory
                         try {
                             refreshWithTimeFeatureFlags(client, featureStore,
                                 StateHolder.getStateFeatureFlag(originEndpoint),
-                                monitor.getFeatureFlagRefreshInterval(), eventData, profiles);
+                                monitor.getFeatureFlagRefreshInterval(), eventData, profiles, replicaLookUp);
                             if (eventData.getDoRefresh()) {
                                 clientFactory.setCurrentConfigStoreClient(originEndpoint, client.getEndpoint());
                                 return eventData;
                             }
                             // If check didn't throw an error other clients don't need to be checked.
                             break;
-                        } catch (AppConfigurationStatusException e) {
+                        } catch (HttpResponseException e) {
                             LOGGER.warn(
                                 "Failed attempting to connect to " + client.getEndpoint() + " during refresh check.");
 
@@ -165,9 +167,9 @@ private static boolean refreshStoreFeatureFlagCheck(FeatureFlagStore featureStor
      * @param eventData Info for this refresh event.
      */
     private static void refreshWithTime(AppConfigurationReplicaClient client, State state, Duration refreshInterval,
-        RefreshEventData eventData) throws AppConfigurationStatusException {
+        RefreshEventData eventData, ReplicaLookUp replicaLookUp) throws AppConfigurationStatusException {
         if (Instant.now().isAfter(state.getNextRefreshCheck())) {
-
+            replicaLookUp.updateAutoFailoverEndpoints();
             refreshWithoutTime(client, state.getWatchKeys(), eventData);
 
             StateHolder.getCurrentState().updateStateRefresh(state, refreshInterval);
@@ -198,11 +200,11 @@ private static void refreshWithoutTime(AppConfigurationReplicaClient client, Lis
     }
 
     private static void refreshWithTimeFeatureFlags(AppConfigurationReplicaClient client, FeatureFlagStore featureStore,
-        State state, Duration refreshInterval, RefreshEventData eventData, List<String> profiles)
+        State state, Duration refreshInterval, RefreshEventData eventData, List<String> profiles, ReplicaLookUp replicaLookUp)
         throws AppConfigurationStatusException {
         Instant date = Instant.now();
         if (date.isAfter(state.getNextRefreshCheck())) {
-
+            replicaLookUp.updateAutoFailoverEndpoints();
             int watchedKeySize = 0;
 
             for (FeatureFlagKeyValueSelector watchKey : featureStore.getSelects()) {

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/implementation/AppConfigurationReplicaClient.java

@@ -116,8 +116,9 @@ List<ConfigurationSetting> listSettings(SettingSelector settingSelector)
         List<ConfigurationSetting> configurationSettings = new ArrayList<>();
         try {
             PagedIterable<ConfigurationSetting> settings = client.listConfigurationSettings(settingSelector);
-            this.failedAttempts = 0;
             settings.forEach(setting -> configurationSettings.add(NormalizeNull.normalizeNullLabel(setting)));
+            // Needs to happen after or we don't know if the request succeeded or failed.
+            this.failedAttempts = 0;
             return configurationSettings;
         } catch (HttpResponseException e) {
             if (e.getResponse() != null) {

sdk/spring/spring-cloud-azure-appconfiguration-config/src/main/java/com/azure/spring/cloud/appconfiguration/config/implementation/AppConfigurationReplicaClientFactory.java

@@ -7,6 +7,7 @@
 import java.util.Map;
 
 import com.azure.spring.cloud.appconfiguration.config.AppConfigurationStoreHealth;
+import com.azure.spring.cloud.appconfiguration.config.implementation.autofailover.ReplicaLookUp;
 import com.azure.spring.cloud.appconfiguration.config.implementation.properties.ConfigStore;
 
 /**
@@ -25,12 +26,12 @@ public class AppConfigurationReplicaClientFactory {
      * @param configStores configuration info for config stores
      */
     public AppConfigurationReplicaClientFactory(AppConfigurationReplicaClientsBuilder clientBuilder,
-        List<ConfigStore> configStores) {
+        List<ConfigStore> configStores, ReplicaLookUp replicaLookUp) {
         this.configStores = configStores;
         if (CONNECTIONS.size() == 0) {
             for (ConfigStore store : configStores) {
-                ConnectionManager manager = new ConnectionManager(clientBuilder, store);
-                CONNECTIONS.put(manager.getOriginEndpoint(), manager);
+                ConnectionManager manager = new ConnectionManager(clientBuilder, store, replicaLookUp);
+                CONNECTIONS.put(manager.getMainEndpoint(), manager);
             }
         }
     }