Skip to content

Commit

Permalink
Merge fd3ef21 into 7ca7f4e
Browse files Browse the repository at this point in the history
  • Loading branch information
@wu-hui
wu-hui authored Jan 29, 2021
2 parents 7ca7f4e + fd3ef21 commit f6379d9
Show file tree
Hide file tree
Showing 5 changed files with 244 additions and 53 deletions.
7 changes: 4 additions & 3 deletions packages/firestore/src/core/bundle_impl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
import { documentKeySet, DocumentKeySet } from '../model/collections';
import { MaybeDocument, NoDocument } from '../model/document';
import { DocumentKey } from '../model/document_key';
import { normalizeNumber } from '../model/normalize';
import {
BundleMetadata as ProtoBundleMetadata,
NamedQuery as ProtoNamedQuery
Expand Down Expand Up @@ -207,7 +208,7 @@ export function bundleInitialProgress(
documentsLoaded: 0,
bytesLoaded: 0,
totalDocuments: metadata.totalDocuments!,
totalBytes: metadata.totalBytes!
totalBytes: normalizeNumber(metadata.totalBytes!)
};
}

Expand All @@ -221,8 +222,8 @@ export function bundleSuccessProgress(
return {
taskState: 'Success',
documentsLoaded: metadata.totalDocuments!,
bytesLoaded: metadata.totalBytes!,
bytesLoaded: normalizeNumber(metadata.totalBytes!),
totalDocuments: metadata.totalDocuments!,
totalBytes: metadata.totalBytes!
totalBytes: normalizeNumber(metadata.totalBytes!)
};
}
2 changes: 1 addition & 1 deletion packages/firestore/src/protos/firestore_bundle_proto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export interface BundleMetadata {
totalDocuments?: number | null;

/** BundleMetadata totalBytes */
totalBytes?: number | null;
totalBytes?: number | string | null;
}

/** Properties of a BundleElement. */
Expand Down
284 changes: 237 additions & 47 deletions packages/firestore/test/integration/api/bundle.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,18 @@ import * as firestore from '@firebase/firestore-types';
import { expect } from 'chai';

import { EventsAccumulator } from '../util/events_accumulator';
import * as firebaseExport from '../util/firebase_export';
import {
apiDescribe,
toDataArray,
withAlternateTestDb,
withTestDb
} from '../util/helpers';

const Blob = firebaseExport.Blob;
const GeoPoint = firebaseExport.GeoPoint;
const Timestamp = firebaseExport.Timestamp;

export const encoder = new TextEncoder();

function verifySuccessProgress(p: firestore.LoadBundleTaskProgress): void {
Expand All @@ -44,57 +49,62 @@ function verifyInProgress(
expect(p.documentsLoaded).to.equal(expectedDocuments);
}

// This template is generated from bundleWithTestDocsAndQueries in '../util/internal_helpsers.ts',
// and manually copied here.
const BUNDLE_TEMPLATE = [
'{"metadata":{"id":"test-bundle","createTime":{"seconds":1001,"nanos":9999},"version":1,"totalDocuments":2,"totalBytes":1503}}',
'{"namedQuery":{"name":"limit","readTime":{"seconds":1000,"nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"FIRST"}}}',
'{"namedQuery":{"name":"limit-to-last","readTime":{"seconds":1000,"nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"LAST"}}}',
'{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","readTime":{"seconds":1000,"nanos":9999},"exists":true}}',
'{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","createTime":{"seconds":1,"nanos":9},"updateTime":{"seconds":1,"nanos":9},"fields":{"k":{"stringValue":"a"},"bar":{"integerValue":1}}}}',
'{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","readTime":{"seconds":1000,"nanos":9999},"exists":true}}',
'{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","createTime":{"seconds":1,"nanos":9},"updateTime":{"seconds":1,"nanos":9},"fields":{"k":{"stringValue":"b"},"bar":{"integerValue":2}}}}'
];
/**
* Returns a valid bundle string from replacing project id in `BUNDLE_TEMPLATE` with the given
* db project id (also recalculate length prefixes).
*/
function bundleString(
db: firestore.FirebaseFirestore,
template: string[]
): string {
const projectId: string = db.app.options.projectId;

// Extract elements from BUNDLE_TEMPLATE and replace the project ID.
const elements = template.map(e => e.replace(/\{0\}/g, projectId));

// Recalculating length prefixes for elements that are not BundleMetadata.
let bundleContent = '';
for (const element of elements.slice(1)) {
const length = encoder.encode(element).byteLength;
bundleContent += `${length}${element}`;
}

// Update BundleMetadata with new totalBytes.
const totalBytes = encoder.encode(bundleContent).byteLength.toString();
const metadata = JSON.parse(elements[0]);
metadata.metadata.totalBytes = totalBytes;
const metadataContent = JSON.stringify(metadata);
const metadataLength = encoder.encode(metadataContent).byteLength;
return `${metadataLength}${metadataContent}${bundleContent}`;
}

apiDescribe('Bundles', (persistence: boolean) => {
// This template is generated from bundleWithTestDocsAndQueries in '../util/internal_helpers.ts',
// and manually copied here.
const BUNDLE_TEMPLATE = [
'{"metadata":{"id":"test-bundle","createTime":{"seconds":1001,"nanos":9999},"version":1,"totalDocuments":2,"totalBytes":"1503"}}',
'{"namedQuery":{"name":"limit","readTime":{"seconds":"1000","nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"FIRST"}}}',
'{"namedQuery":{"name":"limit-to-last","readTime":{"seconds":1000,"nanos":9999},"bundledQuery":{"parent":"projects/{0}/databases/(default)/documents","structuredQuery":{"from":[{"collectionId":"coll-1"}],"orderBy":[{"field":{"fieldPath":"bar"},"direction":"DESCENDING"},{"field":{"fieldPath":"__name__"},"direction":"DESCENDING"}],"limit":{"value":1}},"limitType":"LAST"}}}',
'{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","readTime":{"seconds":"1000","nanos":9999},"exists":true}}',
'{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/a","createTime":{"seconds":1,"nanos":9},"updateTime":{"seconds":"1","nanos":9},"fields":{"k":{"stringValue":"a"},"bar":{"integerValue":1}}}}',
'{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","readTime":{"seconds":1000,"nanos":9999},"exists":true}}',
'{"document":{"name":"projects/{0}/databases/(default)/documents/coll-1/b","createTime":{"seconds":"1","nanos":9},"updateTime":{"seconds":1,"nanos":9},"fields":{"k":{"stringValue":"b"},"bar":{"integerValue":2}}}}'
];

function verifySnapEqualsTestDocs(snap: firestore.QuerySnapshot): void {
expect(toDataArray(snap)).to.deep.equal([
{ k: 'a', bar: 1 },
{ k: 'b', bar: 2 }
]);
}

/**
* Returns a valid bundle string from replacing project id in `BUNDLE_TEMPLATE` with the given
* db project id (also recalculate length prefixes).
*/
function bundleString(db: firestore.FirebaseFirestore): string {
const projectId: string = db.app.options.projectId;

// Extract elements from BUNDLE_TEMPLATE and replace the project ID.
const elements = BUNDLE_TEMPLATE.map(e => e.replace('{0}', projectId));

// Recalculating length prefixes for elements that are not BundleMetadata.
let bundleContent = '';
for (const element of elements.slice(1)) {
const length = encoder.encode(element).byteLength;
bundleContent += `${length}${element}`;
}

// Update BundleMetadata with new totalBytes.
const totalBytes = encoder.encode(bundleContent).byteLength;
const metadata = JSON.parse(elements[0]);
metadata.metadata.totalBytes = totalBytes;
const metadataContent = JSON.stringify(metadata);
const metadataLength = encoder.encode(metadataContent).byteLength;
return `${metadataLength}${metadataContent}${bundleContent}`;
}

it('load with documents only with on progress and promise interface', () => {
return withTestDb(persistence, async db => {
const progressEvents: firestore.LoadBundleTaskProgress[] = [];
let completeCalled = false;
const task: firestore.LoadBundleTask = db.loadBundle(bundleString(db));
const task: firestore.LoadBundleTask = db.loadBundle(
bundleString(db, BUNDLE_TEMPLATE)
);
task.onProgress(
progress => {
progressEvents.push(progress);
Expand Down Expand Up @@ -138,7 +148,7 @@ apiDescribe('Bundles', (persistence: boolean) => {
it('load with documents and queries with promise interface', () => {
return withTestDb(persistence, async db => {
const fulfillProgress: firestore.LoadBundleTaskProgress = await db.loadBundle(
bundleString(db)
bundleString(db, BUNDLE_TEMPLATE)
);

verifySuccessProgress(fulfillProgress!);
Expand All @@ -152,12 +162,12 @@ apiDescribe('Bundles', (persistence: boolean) => {

it('load for a second time skips', () => {
return withTestDb(persistence, async db => {
await db.loadBundle(bundleString(db));
await db.loadBundle(bundleString(db, BUNDLE_TEMPLATE));

let completeCalled = false;
const progressEvents: firestore.LoadBundleTaskProgress[] = [];
const task: firestore.LoadBundleTask = db.loadBundle(
encoder.encode(bundleString(db))
encoder.encode(bundleString(db, BUNDLE_TEMPLATE))
);
task.onProgress(
progress => {
Expand Down Expand Up @@ -194,7 +204,7 @@ apiDescribe('Bundles', (persistence: boolean) => {

const progress = await db.loadBundle(
// Testing passing in non-string bundles.
encoder.encode(bundleString(db))
encoder.encode(bundleString(db, BUNDLE_TEMPLATE))
);

verifySuccessProgress(progress);
Expand All @@ -214,7 +224,7 @@ apiDescribe('Bundles', (persistence: boolean) => {
it('loaded documents should not be GC-ed right away', () => {
return withTestDb(persistence, async db => {
const fulfillProgress: firestore.LoadBundleTaskProgress = await db.loadBundle(
bundleString(db)
bundleString(db, BUNDLE_TEMPLATE)
);

verifySuccessProgress(fulfillProgress!);
Expand All @@ -234,12 +244,14 @@ apiDescribe('Bundles', (persistence: boolean) => {
it('load with documents from other projects fails', () => {
return withTestDb(persistence, async db => {
return withAlternateTestDb(persistence, async otherDb => {
await expect(otherDb.loadBundle(bundleString(db))).to.be.rejectedWith(
'Tried to deserialize key from different project'
);
await expect(
otherDb.loadBundle(bundleString(db, BUNDLE_TEMPLATE))
).to.be.rejectedWith('Tried to deserialize key from different project');

// Verify otherDb still functions, despite loaded a problematic bundle.
const finalProgress = await otherDb.loadBundle(bundleString(otherDb));
const finalProgress = await otherDb.loadBundle(
bundleString(otherDb, BUNDLE_TEMPLATE)
);
verifySuccessProgress(finalProgress);

// Read from cache. These documents do not exist in backend, so they can
Expand All @@ -252,3 +264,181 @@ apiDescribe('Bundles', (persistence: boolean) => {
});
});
});

// TODO(b/178418242): Move this to `api_internal` and generate testing bundle programmatically
// instead of having hard coded bundle strings.
apiDescribe('Bundles conformance', (persistence: boolean) => {
const CONFORMANCE_DOC = {
stringValue: 'a',
trueValue: true,
falseValue: false,
integerValue: 10,
largeIntegerValue: 1234567890000,
doubleValue: 0.1,
infinityValue: Infinity,
negativeInfinityValue: -Infinity,
objectValue: { foo: 'bar', '😀': '😜' },
emptyObject: {},
dateValue: new Timestamp(479978400, 123000000),
zeroDateValue: new Timestamp(0, 0),
arrayValue: ['foo', 42, 'bar'],
emptyArray: [],
nilValue: null,
geoPointValue: new GeoPoint(50.1430847, -122.947778),
zeroGeoPointValue: new GeoPoint(0, 0),
bytesValue: Blob.fromUint8Array(new Uint8Array([0x01, 0x02]))
};

// This is built by Node(protobuf.js) from above document with some mixing of int64 numbers
// represented as both number and string, and one additional field of `pathValue: db.doc('col1/ref1')`.
const BUNDLES_PROTOBUFJS_CONFORMANCE_TEMPLATE = [
'{"metadata":{"id":"test-bundle","createTime":{"seconds":"1611502887","nanos":707859000},"version":1,"totalDocuments":1,"totalBytes":"1405"}}',
'{"documentMetadata":{"name":"projects/{0}/databases/(default)/documents/bundles/conformance-doc","readTime":{"seconds":"1611502887","nanos":707859000},"exists":true}}',
'{"document":{"name":"projects/{0}/databases/(default)/documents/bundles/conformance-doc","fields":{"largeIntegerValue":{"integerValue":"1234567890000"},"bytesValue":{"bytesValue":"AQI="},"trueValue":{"booleanValue":true},"integerValue":{"integerValue":"10"},"zeroDateValue":{"timestampValue":{"seconds":"0","nanos":0}},"doubleValue":{"doubleValue":0.1},"geoPointValue":{"geoPointValue":{"latitude":50.1430847,"longitude":-122.947778}},"objectValue":{"mapValue":{"fields":{"😀":{"stringValue":"😜"},"foo":{"stringValue":"bar"}}}},"zeroGeoPointValue":{"geoPointValue":{"latitude":0,"longitude":0}},"emptyArray":{"arrayValue":{}},"nilValue":{"nullValue":"NULL_VALUE"},"emptyObject":{"mapValue":{}},"infinityValue":{"doubleValue":"Infinity"},"dateValue":{"timestampValue":{"seconds":"479978400","nanos":123000000}},"falseValue":{"booleanValue":false},"pathValue":{"referenceValue":"projects/{0}/databases/(default)/documents/col1/ref1"},"stringValue":{"stringValue":"a"},"negativeInfinityValue":{"doubleValue":"-Infinity"},"arrayValue":{"arrayValue":{"values":[{"stringValue":"foo"},{"integerValue":"42"},{"stringValue":"bar"}]}}},"createTime":{"seconds":"1611502887"},"updateTime":{"seconds":"1611502887"}}}'
];

// This is built by Java(proto3 json formatter) with above document and one additional field of `pathValue: db.doc('col1/ref1')`.
const BUNDLES_PROTOB3_CONFORMANCE_TEMPLATE = [
`{
"metadata": {
"id": "test-bundle",
"createTime": "2021-01-24T20:03:40.608348Z",
"version": 1,
"totalDocuments": 1,
"totalBytes": "2164"
}
}`,
`{
"documentMetadata": {
"name": "projects/{0}/databases/(default)/documents/bundles/conformance-doc",
"readTime": "2021-01-24T20:03:40.608348Z",
"exists": true
}
}`,
`{
"document": {
"name": "projects/{0}/databases/(default)/documents/bundles/conformance-doc",
"fields": {
"integerValue": {
"integerValue": "10"
},
"pathValue": {
"referenceValue": "projects/{0}/databases/(default)/documents/col1/ref1"
},
"geoPointValue": {
"geoPointValue": {
"latitude": 50.1430847,
"longitude": -122.947778
}
},
"falseValue": {
"booleanValue": false
},
"emptyObject": {
"mapValue": {
}
},
"doubleValue": {
"doubleValue": 0.1
},
"emptyArray": {
"arrayValue": {
}
},
"zeroGeoPointValue": {
"geoPointValue": {
}
},
"stringValue": {
"stringValue": "a"
},
"trueValue": {
"booleanValue": true
},
"nilValue": {
"nullValue": null
},
"dateValue": {
"timestampValue": "1985-03-18T07:20:00.123Z"
},
"arrayValue": {
"arrayValue": {
"values": [{
"stringValue": "foo"
}, {
"integerValue": "42"
}, {
"stringValue": "bar"
}]
}
},
"bytesValue": {
"bytesValue": "AQI="
},
"objectValue": {
"mapValue": {
"fields": {
"foo": {
"stringValue": "bar"
},
"😀": {
"stringValue": "😜"
}
}
}
},
"infinityValue": {
"doubleValue": "Infinity"
},
"zeroDateValue": {
"timestampValue": "1970-01-01T00:00:00Z"
},
"negativeInfinityValue": {
"doubleValue": "-Infinity"
},
"largeIntegerValue": {
"integerValue": "1234567890000"
}
},
"createTime": "2021-01-24T15:41:27.634116Z",
"updateTime": "2021-01-24T15:41:27.634116Z"
}
}`
];

it('loads bundle built by protobuf.js as expected', () => {
return withTestDb(persistence, async db => {
const fulfillProgress: firestore.LoadBundleTaskProgress = await db.loadBundle(
bundleString(db, BUNDLES_PROTOBUFJS_CONFORMANCE_TEMPLATE)
);

verifySuccessProgress(fulfillProgress!);

const snap = (
await db.doc('bundles/conformance-doc').get({ source: 'cache' })
).data()!;

expect(snap.pathValue.path).to.equal(db.doc('col1/ref1').path);
delete snap.pathValue;
expect(snap).to.deep.equal(CONFORMANCE_DOC);
});
});

it('loads bundle built by proto3 json output as expected', () => {
return withTestDb(persistence, async db => {
const fulfillProgress: firestore.LoadBundleTaskProgress = await db.loadBundle(
bundleString(db, BUNDLES_PROTOB3_CONFORMANCE_TEMPLATE)
);

verifySuccessProgress(fulfillProgress!);

const snap = (
await db.doc('bundles/conformance-doc').get({ source: 'cache' })
).data()!;

expect(snap.pathValue.path).to.equal(db.doc('col1/ref1').path);
delete snap.pathValue;
expect(snap).to.deep.equal(CONFORMANCE_DOC);
});
});
});
Loading

0 comments on commit f6379d9

Please sign in to comment.