forked from OurHike/OurHike
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy patharchiveDownload.test.ts
More file actions
1988 lines (1677 loc) · 78.4 KB
/
Copy patharchiveDownload.test.ts
File metadata and controls
1988 lines (1677 loc) · 78.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import { get, set, del } from 'idb-keyval'
import {
downloadArchive,
deleteArchive,
ARCHIVE_PARTIAL_KEY,
ARCHIVE_PROGRESS_KEY,
ARCHIVE_SOURCE_KEY,
ARCHIVE_VERSION_KEY,
ArchiveSizeMismatchError,
ArchiveHashMismatchError,
ArchiveTooLargeError,
readArchiveVersion,
} from './archiveDownload'
import { CORRIDOR_ARCHIVE_KEY } from '../map/pmtilesSource'
import { completedMarker, recordCompleted } from './storageHealth'
import { publishedHash } from './dataManifest'
import { Sha256, sha256Hex, type Sha256State } from './sha256'
// The download behind Downloads.tsx's buttons. WIREFRAMES.md `7a` requires a
// failed transfer to RESUME rather than restart, and that promise is the
// whole reason this module is more than a fetch call: re-pulling 300 MB from
// zero because a connection dropped at 90% is exactly the failure someone on
// trailhead wifi cannot afford.
//
// Two traps get specific attention below.
//
// **A server that ignores Range.** Ask for `bytes=N-` and a compliant server
// answers 206 with the remainder. A server that does not support ranges
// answers 200 with the WHOLE file - and appending that to the bytes already
// held produces a corrupt archive of exactly the right length, which passes
// every size check and then renders a broken map. The status code has to be
// checked, not assumed.
//
// **Never destroying a good archive.** Partial bytes live under their own
// key. A failed or aborted attempt must leave both the partial progress and
// any previously-completed archive intact.
vi.mock('idb-keyval', () => ({ get: vi.fn(), set: vi.fn(), del: vi.fn() }))
// The published-hash lookup is mocked rather than served through the fetch
// stub above: what it reads (latest.json under VITE_DATA_BASE_URL, which is
// unset under test) is dataManifest.ts's own subject, tested there. Here the
// interesting variable is only what the bucket claims - a hash, a different
// hash, or no answer at all.
vi.mock('./dataManifest', () => ({ publishedHash: vi.fn() }))
const mockedGet = vi.mocked(get)
const mockedSet = vi.mocked(set)
const mockedDel = vi.mocked(del)
const mockedPublishedHash = vi.mocked(publishedHash)
const URL_ = 'https://cdn.example.org/background.pmtiles'
/** What latest.json calls it - passed explicitly now, never guessed. */
const ARTIFACT = 'background.pmtiles'
/** In-memory stand-in for IndexedDB. */
function withStore(initial: Record<string, unknown> = {}) {
const store: Record<string, unknown> = { ...initial }
mockedGet.mockImplementation(async (key) => store[key as string])
mockedSet.mockImplementation(async (key, value) => {
store[key as string] = value
})
mockedDel.mockImplementation(async (key) => {
delete store[key as string]
})
return store
}
function bytes(...values: number[]) {
return new Uint8Array(values)
}
/** A fetch returning `chunks` as a stream, with the given status/headers. */
function mockFetch({
chunks,
status = 200,
totalBytes,
contentRange,
etag,
}: {
chunks: Uint8Array[]
status?: number
totalBytes?: number
contentRange?: string
etag?: string
}) {
return vi.spyOn(globalThis, 'fetch').mockImplementation(async () => {
const body = new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) controller.enqueue(chunk)
controller.close()
},
})
const headers = new Headers()
const declared = totalBytes ?? chunks.reduce((n, c) => n + c.length, 0)
headers.set('content-length', String(declared))
if (contentRange) headers.set('content-range', contentRange)
if (etag) headers.set('etag', etag)
return new Response(body, { status, headers })
})
}
/**
* The archive the map would read out of this store, assembled the way
* archiveStore.ts assembles it - or undefined where there is not one.
*
* Since #553 a finished archive is a run of segment records named by a
* completion marker, so "is it stored" is no longer a lookup of one key. These
* helpers keep the tests written in terms of the archive rather than the
* layout: what a hiker has is a map, not a record count.
*/
function storedArchive(
store: Record<string, unknown>,
key = CORRIDOR_ARCHIVE_KEY,
): Blob | undefined {
const complete = store[`${key}:complete`] as { generation: number } | undefined
if (complete === undefined) {
// The pre-#553 whole-archive record, which archiveStore.ts still serves.
const legacy = store[key]
return legacy instanceof Blob ? legacy : undefined
}
return new Blob(segmentsIn(store, complete.generation, key))
}
/** One generation's segment records, in order, up to the first gap. */
function segmentsIn(
store: Record<string, unknown>,
generation: number,
key = CORRIDOR_ARCHIVE_KEY,
): Blob[] {
const found: Blob[] = []
for (let index = 0; ; index += 1) {
const part = store[`${key}:g${generation}:${index}`]
if (!(part instanceof Blob)) return found
found.push(part)
}
}
/**
* The bytes a resume would pick up - segments on disk that are NOT the finished
* archive - or undefined where there are none.
*
* Defined by exclusion rather than by reading the source record, because that is
* the question the tests are actually asking: "was anything left behind". After
* a completed download the segments are still there and are the archive itself,
* which is not a partial by any reading.
*/
function heldPartial(
store: Record<string, unknown>,
key = CORRIDOR_ARCHIVE_KEY,
): Blob | undefined {
const complete = store[`${key}:complete`] as { generation: number } | undefined
for (const generation of [0, 1]) {
if (complete?.generation === generation) continue
const parts = segmentsIn(store, generation, key)
if (parts.length > 0) return new Blob(parts)
}
return undefined
}
/**
* Store contents standing in for an interrupted transfer holding `blob`.
*
* Spread into a `withStore` fixture. One segment, which is what a real transfer
* under SEGMENT_BYTES produces, and generation 0, which is where a transfer
* starts when nothing has completed under the key yet.
*/
function partialOf(
blob: Blob,
generation = 0,
key = CORRIDOR_ARCHIVE_KEY,
): Record<string, unknown> {
return { [`${key}:g${generation}:0`]: blob }
}
/** A completed archive holding `blob`, as `markComplete` leaves it: segments
* under a generation, named by a marker. What a phone that finished a download
* on a current build actually has. */
function completedOf(
blob: Blob,
generation = 0,
key = CORRIDOR_ARCHIVE_KEY,
): Record<string, unknown> {
return {
...partialOf(blob, generation, key),
[`${key}:complete`]: {
generation,
segments: 1,
totalBytes: blob.size,
},
}
}
beforeEach(() => {
vi.clearAllMocks()
// The default for every test that is not about verification: no published
// answer, which is what a field-test server or an older release gives, and
// which must leave the download behaving exactly as it did before #197.
mockedPublishedHash.mockResolvedValue(null)
})
afterEach(() => {
vi.restoreAllMocks()
})
describe('downloadArchive — a clean first run', () => {
it('stores the finished archive where the map reads it from', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3), bytes(4, 5, 6)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(storedArchive(store)).toBeInstanceOf(Blob)
})
it('stores every byte it was sent, in order', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3), bytes(4, 5, 6)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = new Uint8Array(await (storedArchive(store) as Blob).arrayBuffer())
expect([...stored]).toEqual([1, 2, 3, 4, 5, 6])
})
it('reports progress as chunks arrive, not only at the end', async () => {
withStore()
mockFetch({ chunks: [bytes(1, 2, 3), bytes(4, 5, 6)] })
const onProgress = vi.fn()
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, {
artifactKey: ARTIFACT,
onProgress,
})
expect(onProgress.mock.calls.length).toBeGreaterThan(1)
expect(onProgress).toHaveBeenLastCalledWith({ receivedBytes: 6, totalBytes: 6 })
})
it('clears the partial state once the archive is complete', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(heldPartial(store)).toBeUndefined()
expect(store[ARCHIVE_PROGRESS_KEY]).toBeUndefined()
})
it('asks for the whole file when nothing is held yet', async () => {
withStore()
const spy = mockFetch({ chunks: [bytes(1, 2, 3)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const init = spy.mock.calls[0][1] as RequestInit | undefined
expect(new Headers(init?.headers).get('range')).toBeNull()
})
})
describe('downloadArchive — resuming', () => {
it('asks only for the bytes it does not already have', async () => {
withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
// A partial written by this module always records where it came from;
// one without it is deliberately discarded rather than resumed onto.
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
})
const spy = mockFetch({
chunks: [bytes(4, 5, 6)],
status: 206,
totalBytes: 3,
contentRange: 'bytes 3-5/6',
})
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const init = spy.mock.calls[0][1] as RequestInit | undefined
expect(new Headers(init?.headers).get('range')).toBe('bytes=3-')
})
it('joins the resumed bytes onto what it already had', async () => {
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
// A partial written by this module always records where it came from;
// one without it is deliberately discarded rather than resumed onto.
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
})
mockFetch({
chunks: [bytes(4, 5, 6)],
status: 206,
totalBytes: 3,
contentRange: 'bytes 3-5/6',
})
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = new Uint8Array(await (storedArchive(store) as Blob).arrayBuffer())
expect([...stored]).toEqual([1, 2, 3, 4, 5, 6])
})
it('counts resumed progress from what was already held, not from zero', async () => {
withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
// A partial written by this module always records where it came from;
// one without it is deliberately discarded rather than resumed onto.
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
})
const onProgress = vi.fn()
mockFetch({
chunks: [bytes(4, 5, 6)],
status: 206,
totalBytes: 3,
contentRange: 'bytes 3-5/6',
})
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, {
artifactKey: ARTIFACT,
onProgress,
})
expect(onProgress).toHaveBeenLastCalledWith({ receivedBytes: 6, totalBytes: 6 })
})
it('starts over safely when the server ignores Range and sends the whole file', async () => {
// The silent-corruption trap: appending a full 200 body to existing
// partial bytes yields a file of plausible length that is entirely wrong.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
// A partial written by this module always records where it came from;
// one without it is deliberately discarded rather than resumed onto.
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
})
mockFetch({ chunks: [bytes(9, 9, 9, 9, 9, 9)], status: 200, totalBytes: 6 })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = new Uint8Array(await (storedArchive(store) as Blob).arrayBuffer())
expect([...stored]).toEqual([9, 9, 9, 9, 9, 9])
})
})
describe('downloadArchive — failure', () => {
it('keeps what it received when the connection drops', async () => {
const store = withStore()
vi.spyOn(globalThis, 'fetch').mockImplementation(async () => {
// Pull-based, so the first read genuinely delivers a chunk and the
// SECOND one fails. Calling controller.error() straight after an
// enqueue in start() discards the queued chunk instead, which models a
// connection that died before delivering anything - a different case,
// and not the one this test is about.
let pulls = 0
const body = new ReadableStream<Uint8Array>({
pull(controller) {
if (pulls === 0) controller.enqueue(bytes(1, 2, 3))
else controller.error(new TypeError('network error'))
pulls += 1
},
})
const headers = new Headers({ 'content-length': '6' })
return new Response(body, { status: 200, headers })
})
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow()
expect(heldPartial(store)).toBeInstanceOf(Blob)
expect(store[ARCHIVE_PROGRESS_KEY]).toMatchObject({ receivedBytes: 3 })
})
it('leaves a previously-downloaded archive untouched when a new attempt fails', async () => {
const store = withStore(completedOf(new Blob([bytes(7, 7, 7)])))
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new TypeError('offline'))
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow()
expect(new Uint8Array(await (storedArchive(store) as Blob).arrayBuffer())).toEqual(
bytes(7, 7, 7),
)
})
it('keeps the archive readable while a re-download is under way', async () => {
// The property generations exist for (#553). Segments live under fixed
// names, so a re-download writing its own segment 0 over the working
// archive's would destroy the map at the moment the hiker asked to update
// it - and it would be gone before a single new byte was verified.
const store = withStore(completedOf(new Blob([bytes(7, 7, 7)])))
let midTransfer: Blob | undefined
vi.spyOn(globalThis, 'fetch').mockImplementation(async () => {
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(bytes(1, 2, 3))
// What the map would read if it drew a tile right now.
midTransfer = storedArchive(store)
controller.error(new TypeError('the connection dropped'))
},
})
return new Response(body, { headers: new Headers({ 'content-length': '6' }) })
})
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow()
expect(new Uint8Array(await (midTransfer as Blob).arrayBuffer())).toEqual(
bytes(7, 7, 7),
)
// And still afterwards: the failed attempt took its own generation with it.
expect(new Uint8Array(await (storedArchive(store) as Blob).arrayBuffer())).toEqual(
bytes(7, 7, 7),
)
})
it('refuses to store an archive shorter than the server said it would be', async () => {
// Truncation is the failure a size check exists to catch: a short PMTiles
// archive opens fine and then returns nothing for tiles past the cut.
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3)], totalBytes: 99 })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toBeInstanceOf(ArchiveSizeMismatchError)
expect(storedArchive(store)).toBeUndefined()
})
it('says the mismatch in a hiker’s units, keeping the raw counts as fields', () => {
// The message IS the UI - the download card renders it in a role="alert" -
// and it used to read "297483822 bytes but the server said 314572800",
// which is a log line shown to the one person guaranteed not to want one.
const error = new ArchiveSizeMismatchError(314_000_000, 297_000_000)
expect(error.message).toMatch(/297 MB/)
expect(error.message).toMatch(/314 MB/)
expect(error.message).not.toMatch(/\d{6,}/)
expect(error.expectedBytes).toBe(314_000_000)
expect(error.actualBytes).toBe(297_000_000)
})
it('keeps the partial bytes after a size mismatch, so a resume can finish it', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3)], totalBytes: 99 })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow()
expect(heldPartial(store)).toBeInstanceOf(Blob)
})
it('stops when aborted, keeping what it has', async () => {
const store = withStore()
const controller = new AbortController()
mockFetch({ chunks: [bytes(1, 2, 3), bytes(4, 5, 6)] })
// Aborted from the progress callback, which fires only after a chunk has
// really been taken in. Triggering it from inside the stream instead raced
// the read loop - the stream refills its queue as `read()` resolves, so the
// abort landed before the first chunk was accounted for, and the assertion
// below then held for an EMPTY partial: "keeping what it has" where it had
// nothing.
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, {
artifactKey: ARTIFACT,
signal: controller.signal,
onProgress: ({ receivedBytes }) => {
if (receivedBytes >= 3) controller.abort()
},
}),
).rejects.toThrow()
// The bytes that arrived before the abort are on disk, so the next attempt
// resumes from there.
expect(new Uint8Array(await (heldPartial(store) as Blob).arrayBuffer())).toEqual(
bytes(1, 2, 3),
)
expect(storedArchive(store)).toBeUndefined()
})
it('rejects a non-OK response without touching stored state', async () => {
const store = withStore()
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(null, { status: 404, statusText: 'Not Found' }),
)
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(/404/)
expect(heldPartial(store)).toBeUndefined()
})
})
// The failure mode the size check is structurally unable to catch. totalBytes is
// DEFINED as heldBytes + declared, and a completed resume accumulates exactly
// heldBytes + declared - both sides of that comparison are the same expression.
// So a spliced archive always passes it, and the result is a PMTiles file whose
// directory and tile offsets disagree: a map that reports itself downloaded and
// renders wrong past the seam, offline, with no network to correct it.
describe('downloadArchive — refusing to splice two different archives', () => {
it('discards partial bytes that came from a different URL', async () => {
// A hiker who starts Standard, fails, then picks Light. Appending z12 bytes
// onto z11 bytes would produce exactly the right length and a broken map.
const store = withStore({
...partialOf(new Blob([bytes(9, 9, 9)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: 'https://cdn.example.org/background_z13.pmtiles' },
})
const fetchSpy = mockFetch({ chunks: [bytes(1, 2, 3)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
// No Range header at all: this is a fresh start, not a resume.
expect(fetchSpy.mock.calls[0][1]).toMatchObject({ headers: undefined })
const stored = storedArchive(store) as Blob
expect(stored.size).toBe(3)
})
it('discards a partial with no source record, rather than resuming onto it blindly', async () => {
// Written by a build before the source record existed, or left behind when
// a quota failure interrupted persistPartial partway through.
const store = withStore({
...partialOf(new Blob([bytes(9, 9, 9)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
})
mockFetch({ chunks: [bytes(1, 2, 3)] })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect((storedArchive(store) as Blob).size).toBe(3)
})
it('sends If-Range so the server itself refuses a stale resume', async () => {
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"' },
})
const fetchSpy = mockFetch({
chunks: [bytes(4, 5, 6)],
status: 206,
etag: '"v1"',
})
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(fetchSpy.mock.calls[0][1]).toMatchObject({
headers: { Range: 'bytes=3-', 'If-Range': '"v1"' },
})
expect((storedArchive(store) as Blob).size).toBe(6)
})
it('starts clean when the archive was republished mid-download', async () => {
// If-Range makes the server arbitrate: a changed object comes back 200 with
// the whole body instead of 206, and the existing status check treats that
// as "start clean". The held bytes must NOT survive into the result.
const store = withStore({
...partialOf(new Blob([bytes(9, 9, 9)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"' },
})
mockFetch({ chunks: [bytes(1, 2, 3, 4)], status: 200, etag: '"v2"' })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = storedArchive(store) as Blob
expect(stored.size).toBe(4)
expect(new Uint8Array(await stored.arrayBuffer())).toEqual(bytes(1, 2, 3, 4))
})
it('refuses a 206 whose ETag is not the one those bytes came from', async () => {
// THE CASE THE r2.dev BUCKET ACTUALLY PRODUCES (#506). The test above has
// the server arbitrating correctly - stale If-Range, 200, whole body. This
// one has it ignoring If-Range and serving the range anyway, which is what
// was measured against the live bucket: a stale validator is answered 206.
//
// The 206 is therefore worthless as evidence and the ETag is the only thing
// left saying which object these bytes are from. Without the comparison,
// bytes 4,5,6 of a DIFFERENT archive get appended to a held 1,2,3 and the
// result is a 6-byte file of exactly the expected length - the splice that
// renders a wrong map past the seam with no network to correct it.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 0, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5, 6)], status: 206, etag: '"v2"' })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(/no longer matches what the server has/)
// Nothing spliced was stored, and the unusable segments are gone rather than
// left for the next attempt to append to all over again.
expect(storedArchive(store)).toBeUndefined()
expect(heldPartial(store)).toBeUndefined()
expect(store[ARCHIVE_SOURCE_KEY]).toBeUndefined()
expect(store[ARCHIVE_PROGRESS_KEY]).toBeUndefined()
})
it('leaves a previously-downloaded map alone when it refuses a stale 206', async () => {
// The rule the whole module is built around: someone with a good map who
// taps update and hits a republished archive still has their good map.
// The good map is a completed archive in generation 0, so the interrupted
// transfer is in generation 1 - which is what #553's generations are for,
// and what makes "refusing the stale 206 does not touch it" a real claim
// rather than one the fixture arranges by using a different record name.
const store = withStore({
...completedOf(new Blob([bytes(7, 7, 7, 7, 7, 7)]), 0),
...partialOf(new Blob([bytes(1, 2, 3)]), 1),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 1, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5, 6)], status: 206, etag: '"v2"' })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(/no longer matches what the server has/)
const good = storedArchive(store) as Blob
expect(good.size).toBe(6)
expect(new Uint8Array(await good.arrayBuffer())).toEqual(bytes(7, 7, 7, 7, 7, 7))
})
it('resumes onto a 206 that states the same strong ETag', async () => {
// The other half of the discrimination: the check must refuse a CHANGED
// object without refusing the ordinary resume, which is the case the
// WIREFRAMES 7a promise is made of.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 0, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5, 6)], status: 206, etag: '"v1"' })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = storedArchive(store) as Blob
expect(new Uint8Array(await stored.arrayBuffer())).toEqual(bytes(1, 2, 3, 4, 5, 6))
})
it('resumes when the 206 states no ETag at all, leaving the hash to decide', async () => {
// A bucket that exposes no ETag is the case PartialSource.etag is optional
// for. There is nothing to compare, so this must not become a refusal to
// resume - the published-hash check is what covers that configuration, and
// it still runs at the end.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 0, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5, 6)], status: 206 })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(storedArchive(store)?.size).toBe(6)
})
it('resumes when the 206 states only a weak ETag, which cannot validate a range', async () => {
// A weak validator promises semantic equivalence, not byte identity, so it
// is not evidence the object CHANGED any more than it is evidence it did
// not. Refusing on one would strand a resume on a server that sends them.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 6 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 0, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5, 6)], status: 206, etag: 'W/"v2"' })
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(storedArchive(store)?.size).toBe(6)
})
it('keeps the strong ETag it was holding when a resume is interrupted under a weak one', async () => {
// The label a partial carries has to stay the strong validator those bytes
// were held against. Overwriting it with a weak one would leave the next
// attempt comparing against something that cannot arbitrate a range, which
// is the check above quietly disarmed. Asserted on an interrupted transfer
// because a completed one discards the record it would be read from.
const store = withStore({
...partialOf(new Blob([bytes(1, 2, 3)])),
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 3, totalBytes: 9 },
[ARCHIVE_SOURCE_KEY]: { url: URL_, etag: '"v1"', generation: 0, segments: 1 },
})
mockFetch({ chunks: [bytes(4, 5)], status: 206, totalBytes: 99, etag: 'W/"v2"' })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveSizeMismatchError)
expect(store[ARCHIVE_SOURCE_KEY]).toMatchObject({ etag: '"v1"' })
})
it('records the source alongside the bytes when a transfer is interrupted', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2)], totalBytes: 99, etag: '"v1"' })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveSizeMismatchError)
// generation and segments are the layout #553 added: which run of segment
// records these bytes are in, and how many of them there are.
expect(store[ARCHIVE_SOURCE_KEY]).toEqual({
url: URL_,
etag: '"v1"',
generation: 0,
segments: 1,
})
})
it('ignores a weak ETag, which does not promise the byte identity a resume needs', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2)], totalBytes: 99, etag: 'W/"v1"' })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveSizeMismatchError)
expect(store[ARCHIVE_SOURCE_KEY]).toEqual({
url: URL_,
etag: undefined,
generation: 0,
segments: 1,
})
})
describe('downloadArchive — a response with nothing to read', () => {
it('fails loudly rather than storing an empty archive', async () => {
// A 200 with a null body is not a zero-byte map, it is a broken response.
// Writing it to CORRIDOR_ARCHIVE_KEY would leave the app convinced it has
// a map, offline, with no way to find out otherwise.
const store = withStore()
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(null, { status: 200, headers: { 'content-length': '3' } }),
)
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(/no map data arrived/)
expect(storedArchive(store)).toBeUndefined()
})
})
describe('downloadArchive — a server that never says how big the file is', () => {
it('still stores what arrived, rather than treating no length as zero length', async () => {
// A chunked response carries no content-length. There is nothing to check
// the size against, so the length check has to stand down instead of
// deciding the archive is short - refusing a complete download over a
// header the server was never obliged to send would be its own bug.
const store = withStore()
vi.spyOn(globalThis, 'fetch').mockImplementation(
async () =>
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(bytes(1, 2, 3, 4))
controller.close()
},
}),
{ status: 200 },
),
)
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
const stored = storedArchive(store) as Blob
expect(stored.size).toBe(4)
expect(heldPartial(store)).toBeUndefined()
})
it('reports progress against an unknown total rather than a made-up one', async () => {
withStore()
vi.spyOn(globalThis, 'fetch').mockImplementation(
async () =>
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(bytes(1, 2))
controller.close()
},
}),
{ status: 200 },
),
)
const onProgress = vi.fn()
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, {
artifactKey: ARTIFACT,
onProgress,
})
expect(onProgress).toHaveBeenCalledWith({ receivedBytes: 2, totalBytes: 0 })
})
})
})
describe('downloadArchive — packages are independent (issue #200)', () => {
// The multi-package guarantee in one sentence: every record a download
// touches derives from its own package key, so nothing one package does -
// succeed, fail mid-stream, or get deleted - can reach another package's
// bytes. These tests run two packages through exactly those lifecycles.
const OTHER_KEY = 'ourhike:test-dem'
const OTHER_URL = 'https://cdn.example.org/dem.pmtiles'
it("downloading one package leaves another package's archive and partial untouched", async () => {
const corridorPartial = new Blob(['corridor partial'])
const store = withStore({
...partialOf(corridorPartial),
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
[ARCHIVE_PROGRESS_KEY]: { receivedBytes: 16, totalBytes: 100 },
})
mockFetch({ chunks: [bytes(9, 9, 9)] })
await downloadArchive(OTHER_KEY, OTHER_URL, { artifactKey: 'dem.pmtiles' })
expect(storedArchive(store, OTHER_KEY)).toBeInstanceOf(Blob)
// The corridor package's resumable state survives, byte for byte.
expect(await (heldPartial(store) as Blob).text()).toBe('corridor partial')
expect(store[ARCHIVE_SOURCE_KEY]).toEqual({ url: URL_ })
expect(store[ARCHIVE_PROGRESS_KEY]).toEqual({ receivedBytes: 16, totalBytes: 100 })
})
it("a failed download persists its partial under its own package, not another's", async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2)], totalBytes: 10 })
await expect(
downloadArchive(OTHER_KEY, OTHER_URL, { artifactKey: 'dem.pmtiles' }),
).rejects.toThrow()
expect(heldPartial(store, OTHER_KEY)).toBeInstanceOf(Blob)
expect(heldPartial(store)).toBeUndefined()
})
it("deleting one package keeps every other package's archive", async () => {
const store = withStore({
...completedOf(new Blob(['corridor'])),
...completedOf(new Blob(['dem']), 0, OTHER_KEY),
...partialOf(new Blob(['stale attempt']), 1, OTHER_KEY),
})
await deleteArchive(OTHER_KEY)
expect(storedArchive(store, OTHER_KEY)).toBeUndefined()
expect(heldPartial(store, OTHER_KEY)).toBeUndefined()
expect(await (storedArchive(store) as Blob).text()).toBe('corridor')
})
it('resume identity is judged per package: the same URL on another package starts clean', async () => {
// A partial held by the corridor package must not be resumed onto by a
// different package downloading from the same URL - the records simply
// never meet, because the keys differ.
const store = withStore({
...partialOf(new Blob(['held'])),
[ARCHIVE_SOURCE_KEY]: { url: URL_ },
})
const spy = mockFetch({ chunks: [bytes(5)] })
await downloadArchive(OTHER_KEY, URL_, { artifactKey: ARTIFACT })
const init = spy.mock.calls[0][1] as RequestInit | undefined
expect(init?.headers).toBeUndefined()
expect(heldPartial(store)).toBeInstanceOf(Blob)
})
})
describe('the completion marker (#190)', () => {
// Written on success, cleared on delete, and in localStorage rather than
// IndexedDB - the whole point is surviving the store the archive did not.
beforeEach(() => {
localStorage.clear()
})
afterEach(() => {
localStorage.clear()
})
it('records a completed download, after the bytes are really stored', async () => {
withStore()
mockFetch({ chunks: [bytes(1, 2, 3)] })
expect(completedMarker(CORRIDOR_ARCHIVE_KEY)).toBeNull()
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(completedMarker(CORRIDOR_ARCHIVE_KEY)).toBeInstanceOf(Date)
})
it('does not record an attempt that failed short', async () => {
// A marker without a completed archive is exactly the false eviction
// claim the marker must never produce.
withStore()
mockFetch({ chunks: [bytes(1)], totalBytes: 3 })
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow()
expect(completedMarker(CORRIDOR_ARCHIVE_KEY)).toBeNull()
})
it('clears the marker when the hiker deletes the archive', async () => {
// "The phone removed your map" about a deletion they performed would be
// the marker lying in the other direction.
withStore({ [CORRIDOR_ARCHIVE_KEY]: new Blob(['x']) })
recordCompleted(CORRIDOR_ARCHIVE_KEY)
await deleteArchive(CORRIDOR_ARCHIVE_KEY)
expect(completedMarker(CORRIDOR_ARCHIVE_KEY)).toBeNull()
})
})
describe('verification against the published hash (#197)', () => {
// The failure being closed here is not a download that breaks - it is one
// that succeeds and is wrong. Bytes from two builds spliced at the resume
// point produce a file of exactly the expected length, so the size check
// provably cannot see it (totalBytes is DEFINED as heldBytes + declared),
// and what arrives is a PMTiles archive whose directory disagrees with its
// tiles: a map that reports itself downloaded and renders wrong past the
// seam, with no network to correct it.
const HELD = bytes(1, 2, 3, 4)
const REST = bytes(5, 6, 7, 8)
const WHOLE = bytes(1, 2, 3, 4, 5, 6, 7, 8)
/** The hash of a resume that went right: held bytes then the remainder. */
const WHOLE_HASH = sha256Hex(WHOLE)
it('stores an archive whose bytes hash to what was published', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3), bytes(4, 5, 6)] })
mockedPublishedHash.mockResolvedValue(sha256Hex(bytes(1, 2, 3, 4, 5, 6)))
await downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT })
expect(storedArchive(store)).toBeInstanceOf(Blob)
})
it('keeps nothing when the completed bytes are not what was published', async () => {
const store = withStore()
mockFetch({ chunks: [bytes(1, 2, 3)] })
mockedPublishedHash.mockResolvedValue(sha256Hex(bytes(9, 9, 9)))
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveHashMismatchError)
// Not stored, and not left resumable either: the bytes are the right
// length and the wrong file, so resuming onto them would only rebuild
// the same wrong archive.
expect(storedArchive(store)).toBeUndefined()
expect(heldPartial(store)).toBeUndefined()
expect(store[ARCHIVE_PROGRESS_KEY]).toBeUndefined()
expect(store[ARCHIVE_SOURCE_KEY]).toBeUndefined()
})
it('leaves a working archive alone when an update fails verification', async () => {
const working = new Blob(['the map that already works'])
const store = withStore(completedOf(working))
mockFetch({ chunks: [bytes(1, 2, 3)] })
mockedPublishedHash.mockResolvedValue(sha256Hex(bytes(9, 9, 9)))
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveHashMismatchError)
expect(await (storedArchive(store) as Blob).text()).toBe('the map that already works')
})
it('catches a splice that completes to exactly the expected length', async () => {
// The case the length check cannot reach. Four held bytes from one build,
// four more from another; the server honours the range (206) and the
// total is exactly what was promised. Only the digest can tell.
const store = withStore({
...partialOf(new Blob([HELD])),
[ARCHIVE_SOURCE_KEY]: { url: URL_, sha256: WHOLE_HASH },
})
mockFetch({ chunks: [bytes(90, 91, 92, 93)], status: 206, totalBytes: 4 })
mockedPublishedHash.mockResolvedValue(WHOLE_HASH)
await expect(
downloadArchive(CORRIDOR_ARCHIVE_KEY, URL_, { artifactKey: ARTIFACT }),
).rejects.toThrow(ArchiveHashMismatchError)
expect(storedArchive(store)).toBeUndefined()
expect(heldPartial(store)).toBeUndefined()
})
it('completes a resume whose bytes really do belong together', async () => {
const store = withStore({