Use exponential backoff for sync retry to improve latency instead of waiting 2s between attempts

This commit is contained in:
Michael Derynck 2022-10-04 15:27:24 -06:00
parent e695093c6c
commit f56d82c5db
3 changed files with 64 additions and 52 deletions

View file

@ -20,7 +20,13 @@ import Text from 'components/Text/Text';
import WithConfirm from 'components/WithConfirm/WithConfirm';
import logo from 'img/logo.svg';
import { makeRequest } from 'network';
import { createGrafanaToken, getPluginSyncStatus, startPluginSync, updateGrafanaToken } from 'state/plugin';
import {
createGrafanaToken,
getPluginSyncStatus,
startPluginSync, SYNC_STATUS_RETRY_LIMIT,
syncStatusDelay,
updateGrafanaToken
} from 'state/plugin';
import { GRAFANA_LICENSE_OSS } from 'utils/consts';
import { getItem, setItem } from 'utils/localStorage';
@ -178,38 +184,33 @@ export const PluginConfigPage = (props: Props) => {
setPluginConfigLoading(false);
}, []);
const waitForSyncStatus = (retryCount = 0) => {
if (retryCount > SYNC_STATUS_RETRY_LIMIT) {
setPluginStatusMessage(
`OnCall took too many tries to synchronize. Did you launch Celery workers? Background workers should perform synchronization, not web server.`
);
setRetrySync(true);
setPluginStatusOk(false);
setPluginConfigLoading(false);
return;
}
getPluginSyncStatus().then((get_sync_response) => {
if (get_sync_response.hasOwnProperty('token_ok')) {
finishSync(get_sync_response);
} else {
syncStatusDelay(retryCount + 1).then(() => waitForSyncStatus(retryCount + 1))
}
}).catch((e) => {
handleSyncException(e);
});
}
const startSync = useCallback(() => {
setRetrySync(false);
setPluginConfigLoading(true);
startPluginSync()
.then(() => {
let counter = 0;
const interval = setInterval(() => {
counter++;
getPluginSyncStatus()
.then((get_sync_response) => {
if (get_sync_response.hasOwnProperty('token_ok')) {
clearInterval(interval);
finishSync(get_sync_response);
}
})
.catch((e) => {
clearInterval(interval);
handleSyncException(e);
});
if (counter >= 5) {
clearInterval(interval);
setPluginStatusMessage(
`OnCall took too many tries to synchronize. Did you launch Celery workers? Background workers should perform synchronization, not web server.`
);
setRetrySync(true);
setPluginStatusOk(false);
setPluginConfigLoading(false);
}
}, 2000);
})
.then(() => waitForSyncStatus())
.catch(handleSyncException);
}, []);

View file

@ -31,6 +31,11 @@ export async function startPluginSync() {
return await makeRequest('/plugin/sync', { method: 'POST' });
}
export const SYNC_STATUS_RETRY_LIMIT = 10;
export const syncStatusDelay = retryCount => new Promise(resolve => setTimeout(resolve, 10 * 2 ** retryCount));
export async function getPluginSyncStatus() {
return await makeRequest(`/plugin/sync`, { method: 'GET' });
}

View file

@ -31,7 +31,14 @@ import { UserGroupStore } from 'models/user_group/user_group';
import { makeRequest } from 'network';
import { AppFeature } from './features';
import { createGrafanaToken, getPluginSyncStatus, installPlugin, startPluginSync, updateGrafanaToken } from './plugin';
import {
createGrafanaToken,
getPluginSyncStatus,
installPlugin,
startPluginSync,
SYNC_STATUS_RETRY_LIMIT, syncStatusDelay,
updateGrafanaToken
} from './plugin';
import { UserAction } from './userAction';
// ------ Dashboard ------ //
@ -182,6 +189,26 @@ export class RootBaseStore {
this.isUserAnonymous = false;
}
async waitForSyncStatus(retryCount = 0) {
if (retryCount > SYNC_STATUS_RETRY_LIMIT) {
this.retrySync = true;
return;
}
getPluginSyncStatus().then((get_sync_response) => {
if (get_sync_response.hasOwnProperty('token_ok')) {
this.finishSync(get_sync_response);
} else {
syncStatusDelay(retryCount + 1)
.then(() => this.waitForSyncStatus(retryCount + 1))
}
}).catch((e) => {
this.handleSyncException(e);
});
}
async setupPlugin(meta: AppPluginMeta<OnCallAppSettings>) {
this.resetStatusToDefault();
@ -208,28 +235,7 @@ export class RootBaseStore {
}
await installPlugin();
}
let counter = 0;
const interval = setInterval(() => {
counter++;
getPluginSyncStatus()
.then((get_sync_response) => {
if (get_sync_response.hasOwnProperty('token_ok')) {
clearInterval(interval);
this.finishSync(get_sync_response);
}
})
.catch((e) => {
clearInterval(interval);
this.handleSyncException(e);
});
if (counter >= 10) {
clearInterval(interval);
this.retrySync = true;
}
}, 2000);
await this.waitForSyncStatus();
}
isUserActionAllowed(action: UserAction) {