This commit is contained in:
Bastian Wagner
2026-06-17 10:45:09 +02:00
parent 38141c0358
commit c94a02e6d0
51 changed files with 4220 additions and 628 deletions

View File

@@ -7,6 +7,7 @@ import { StravaRateLimitError } from './strava-rate-limit.error';
import {
StravaActivityPayload,
StravaStreamPayload,
StravaStreamsResponse,
StravaTokenPayload,
} from './strava.types';
@@ -84,15 +85,17 @@ export class StravaClientService {
accessToken: string,
stravaActivityId: string,
): Promise<StravaStreamPayload[]> {
return this.request<StravaStreamPayload[]>({
const streams = await this.request<StravaStreamsResponse>({
method: 'GET',
url: `https://www.strava.com/api/v3/activities/${stravaActivityId}/streams`,
headers: this.authHeaders(accessToken),
params: {
keys: 'time,distance,latlng,altitude,velocity_smooth,heartrate,cadence,watts,temp,moving,grade_smooth',
key_by_type: false,
key_by_type: true,
},
});
return this.normalizeStreamsResponse(streams);
}
private async postToken(
@@ -149,6 +152,22 @@ export class StravaClientService {
return { Authorization: `Bearer ${accessToken}` };
}
private normalizeStreamsResponse(
streams: StravaStreamsResponse,
): StravaStreamPayload[] {
if (Array.isArray(streams)) {
return streams;
}
return Object.entries(streams).map(([type, stream]) => ({
type: stream.type ?? type,
data: Array.isArray(stream.data) ? stream.data : [],
series_type: stream.series_type,
original_size: stream.original_size,
resolution: stream.resolution,
}));
}
private required(configService: ConfigService, key: string): string {
const value = configService.get<string>(key);
if (!value) {

View File

@@ -0,0 +1,54 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
import { Repository } from 'typeorm';
import {
StravaActivityEntity,
StravaActivityStreamPointEntity,
} from '../database/entities';
import { StravaClientService } from './strava-client.service';
import { StravaStreamNormalizerService } from './strava-stream-normalizer.service';
import { StravaTokenService } from './strava-token.service';
@Injectable()
export class StravaStreamImportService {
constructor(
@InjectRepository(StravaActivityStreamPointEntity)
private readonly streamPointRepository: Repository<StravaActivityStreamPointEntity>,
private readonly stravaTokenService: StravaTokenService,
private readonly stravaClientService: StravaClientService,
private readonly streamNormalizer: StravaStreamNormalizerService,
) {}
async importStreamsForActivity(
activity: StravaActivityEntity,
): Promise<number> {
const accessToken = await this.stravaTokenService.getValidAccessToken(
activity.stravaAthleteId,
);
const streams = await this.stravaClientService.getActivityStreams(
accessToken,
activity.stravaActivityId,
);
const points = this.streamNormalizer.normalize(activity.id, streams);
await this.streamPointRepository.delete({ activityId: activity.id });
await this.insertStreamPoints(points);
return points.length;
}
private async insertStreamPoints(
points: Partial<StravaActivityStreamPointEntity>[],
): Promise<void> {
const chunkSize = 1000;
for (let index = 0; index < points.length; index += chunkSize) {
await this.streamPointRepository.insert(
points.slice(
index,
index + chunkSize,
) as QueryDeepPartialEntity<StravaActivityStreamPointEntity>[],
);
}
}
}

View File

@@ -11,6 +11,11 @@ export class StravaSyncController {
return this.stravaSyncService.startSync();
}
@Get('jobs/latest')
getLatestJob(): Promise<StravaSyncJobEntity | null> {
return this.stravaSyncService.getLatestJob();
}
@Get('jobs/:id')
getJob(@Param('id') id: string): Promise<StravaSyncJobEntity> {
return this.stravaSyncService.getJob(id);

View File

@@ -0,0 +1,69 @@
import { StravaSyncJobEntity } from '../database/entities';
import { StravaRateLimitError } from './strava-rate-limit.error';
import { StravaSyncService } from './strava-sync.service';
describe('StravaSyncService', () => {
afterEach(() => {
jest.useRealTimers();
jest.restoreAllMocks();
});
it('keeps a rate limited job visible and retries it after 15 minutes', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-06-16T12:00:00.000Z'));
const job = {
id: 'job-1',
stravaAthleteId: 'athlete-1',
status: 'queued',
activityCount: 0,
detailCount: 0,
streamPointCount: 0,
errorMessage: null,
retryAfter: null,
startedAt: null,
finishedAt: null,
items: [],
} as StravaSyncJobEntity;
const jobRepository = {
findOne: jest.fn().mockResolvedValue(job),
save: jest.fn((entity: Partial<StravaSyncJobEntity>) => {
Object.assign(job, entity);
return Promise.resolve(job);
}),
};
const stravaTokenService = {
getValidAccessToken: jest.fn().mockResolvedValue('access-token'),
};
const stravaClientService = {
listActivities: jest
.fn()
.mockRejectedValueOnce(new StravaRateLimitError('limit', null))
.mockResolvedValueOnce([]),
};
const service = new StravaSyncService(
{} as never,
{} as never,
jobRepository as never,
{} as never,
stravaTokenService as never,
stravaClientService as never,
{} as never,
);
await service.runJob(job.id);
expect(job.status).toBe('rate_limited');
expect(job.retryAfter?.toISOString()).toBe('2026-06-16T12:15:00.000Z');
expect(job.finishedAt).toBeNull();
expect(jest.getTimerCount()).toBe(1);
await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
expect(stravaClientService.listActivities).toHaveBeenCalledTimes(2);
expect(job.status).toBe('completed');
expect(job.finishedAt).toBeInstanceOf(Date);
});
});

View File

@@ -1,10 +1,8 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import { Injectable, NotFoundException, OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
import { In, Repository } from 'typeorm';
import {
StravaActivityEntity,
StravaActivityStreamPointEntity,
StravaAthleteEntity,
StravaSyncJobEntity,
StravaSyncJobItemEntity,
@@ -12,30 +10,52 @@ import {
import { mapStravaActivity } from './strava-activity.mapper';
import { StravaClientService } from './strava-client.service';
import { StravaRateLimitError } from './strava-rate-limit.error';
import { StravaStreamNormalizerService } from './strava-stream-normalizer.service';
import { StravaStreamImportService } from './strava-stream-import.service';
import { StravaTokenService } from './strava-token.service';
import { StravaActivityPayload } from './strava.types';
@Injectable()
export class StravaSyncService {
export class StravaSyncService implements OnModuleInit {
private readonly retryDelayMs = 15 * 60 * 1000;
private readonly retryTimers = new Map<string, NodeJS.Timeout>();
constructor(
@InjectRepository(StravaAthleteEntity)
private readonly athleteRepository: Repository<StravaAthleteEntity>,
@InjectRepository(StravaActivityEntity)
private readonly activityRepository: Repository<StravaActivityEntity>,
@InjectRepository(StravaActivityStreamPointEntity)
private readonly streamPointRepository: Repository<StravaActivityStreamPointEntity>,
@InjectRepository(StravaSyncJobEntity)
private readonly jobRepository: Repository<StravaSyncJobEntity>,
@InjectRepository(StravaSyncJobItemEntity)
private readonly jobItemRepository: Repository<StravaSyncJobItemEntity>,
private readonly stravaTokenService: StravaTokenService,
private readonly stravaClientService: StravaClientService,
private readonly streamNormalizer: StravaStreamNormalizerService,
private readonly stravaStreamImportService: StravaStreamImportService,
) {}
async onModuleInit(): Promise<void> {
const waitingJobs = await this.jobRepository.find({
where: { status: 'rate_limited' },
});
waitingJobs.forEach((job) => this.scheduleRetry(job));
}
async startSync(): Promise<StravaSyncJobEntity> {
const athlete = await this.resolveAthlete();
const activeJob = await this.jobRepository.findOne({
where: {
stravaAthleteId: athlete.id,
status: In(['queued', 'running', 'rate_limited']),
},
order: { createdAt: 'DESC' },
});
if (activeJob) {
this.scheduleRetry(activeJob);
return activeJob;
}
const job = await this.jobRepository.save(
this.jobRepository.create({
stravaAthleteId: athlete.id,
@@ -48,6 +68,22 @@ export class StravaSyncService {
return job;
}
async getLatestJob(): Promise<StravaSyncJobEntity | null> {
const athlete = await this.resolveAthlete();
const job = await this.jobRepository.findOne({
where: { stravaAthleteId: athlete.id },
relations: { items: true },
order: { createdAt: 'DESC' },
});
if (job?.status === 'rate_limited') {
this.scheduleRetry(job);
}
return job;
}
async getJob(jobId: string): Promise<StravaSyncJobEntity> {
const job = await this.jobRepository.findOne({
where: { id: jobId },
@@ -61,9 +97,14 @@ export class StravaSyncService {
}
async runJob(jobId: string): Promise<void> {
this.clearRetry(jobId);
const job = await this.getJob(jobId);
job.status = 'running';
job.startedAt = new Date();
job.finishedAt = null;
job.retryAfter = null;
job.errorMessage = null;
await this.jobRepository.save(job);
try {
@@ -91,6 +132,7 @@ export class StravaSyncService {
job.status = 'completed';
job.finishedAt = new Date();
job.retryAfter = null;
await this.jobRepository.save(job);
} catch (error) {
await this.failJob(job, error);
@@ -105,6 +147,10 @@ export class StravaSyncService {
const stravaActivityId = String(summary.id);
const item = await this.getOrCreateJobItem(job.id, stravaActivityId);
if (item.status === 'completed') {
return;
}
try {
const existingSummary = await this.activityRepository.findOne({
where: {
@@ -126,14 +172,8 @@ export class StravaSyncService {
);
job.detailCount += 1;
const streams = await this.stravaClientService.getActivityStreams(
accessToken,
stravaActivityId,
);
await this.streamPointRepository.delete({ activityId: activity.id });
const points = this.streamNormalizer.normalize(activity.id, streams);
await this.insertStreamPoints(points);
job.streamPointCount += points.length;
job.streamPointCount +=
await this.stravaStreamImportService.importStreamsForActivity(activity);
item.status = 'completed';
item.errorMessage = null;
@@ -151,20 +191,6 @@ export class StravaSyncService {
}
}
private async insertStreamPoints(
points: Partial<StravaActivityStreamPointEntity>[],
): Promise<void> {
const chunkSize = 1000;
for (let index = 0; index < points.length; index += chunkSize) {
await this.streamPointRepository.insert(
points.slice(
index,
index + chunkSize,
) as QueryDeepPartialEntity<StravaActivityStreamPointEntity>[],
);
}
}
private async getOrCreateJobItem(
jobId: string,
stravaActivityId: string,
@@ -173,6 +199,10 @@ export class StravaSyncService {
where: { jobId, stravaActivityId },
});
if (existing) {
if (existing.status === 'completed') {
return existing;
}
existing.status = 'pending';
existing.errorMessage = null;
return this.jobItemRepository.save(existing);
@@ -203,16 +233,51 @@ export class StravaSyncService {
job: StravaSyncJobEntity,
error: unknown,
): Promise<void> {
job.status =
error instanceof StravaRateLimitError ? 'rate_limited' : 'failed';
const isRateLimit = error instanceof StravaRateLimitError;
job.status = isRateLimit ? 'rate_limited' : 'failed';
job.errorMessage = this.errorMessage(error);
job.retryAfter =
error instanceof StravaRateLimitError ? error.retryAfter : null;
job.finishedAt = new Date();
job.retryAfter = isRateLimit
? new Date(Date.now() + this.retryDelayMs)
: null;
job.finishedAt = isRateLimit ? null : new Date();
await this.jobRepository.save(job);
if (isRateLimit) {
this.scheduleRetry(job);
}
}
private errorMessage(error: unknown): string {
return error instanceof Error ? error.message : 'Unknown sync error';
}
private scheduleRetry(job: StravaSyncJobEntity): void {
if (job.status !== 'rate_limited' || this.retryTimers.has(job.id)) {
return;
}
const retryAfter =
job.retryAfter instanceof Date
? job.retryAfter
: new Date(Date.now() + this.retryDelayMs);
const delay = Math.max(retryAfter.getTime() - Date.now(), 0);
const timer = setTimeout(() => {
this.retryTimers.delete(job.id);
void this.runJob(job.id);
}, delay);
timer.unref?.();
this.retryTimers.set(job.id, timer);
}
private clearRetry(jobId: string): void {
const timer = this.retryTimers.get(jobId);
if (!timer) {
return;
}
clearTimeout(timer);
this.retryTimers.delete(jobId);
}
}

View File

@@ -12,6 +12,7 @@ import {
import { StravaAuthController } from './strava-auth.controller';
import { StravaAuthService } from './strava-auth.service';
import { StravaClientService } from './strava-client.service';
import { StravaStreamImportService } from './strava-stream-import.service';
import { StravaStreamNormalizerService } from './strava-stream-normalizer.service';
import { StravaSyncController } from './strava-sync.controller';
import { StravaSyncService } from './strava-sync.service';
@@ -37,8 +38,9 @@ import { TokenCryptoService } from './token-crypto.service';
StravaTokenService,
TokenCryptoService,
StravaStreamNormalizerService,
StravaStreamImportService,
StravaSyncService,
],
exports: [StravaStreamNormalizerService],
exports: [StravaStreamNormalizerService, StravaStreamImportService],
})
export class StravaModule {}

View File

@@ -64,6 +64,10 @@ export interface StravaStreamPayload {
resolution?: string;
}
export type StravaStreamsResponse =
| StravaStreamPayload[]
| Record<string, Omit<StravaStreamPayload, 'type'> & { type?: string }>;
export interface StravaRateLimit {
limit?: string;
usage?: string;