I'm writing a function which iterate through a list of objects and should return: true if all the items have the attributes (cp and type). false if all the items haven't the attributes (cp and type). throw an error if one of the attribute is missed in the object.
const checkList = (list) => {
if (list && !list.length) return false
console.time('setCheckList')
const dict = {
withCpCount: 0,
withoutCpCount: 0,
total: 0
}
for (elem of list) {
const {cp, type} = elem
if((!cp && type) || (cp && !type)) {
throw new Error('The cp/type is not exclusive')
}
if (cp && type && dict.withCpCount === dict.total) {
dict.withCpCount += 1
}
if (!cp && !type && dict.total === dict.withoutCpCount){
dict.withoutCpCount += 1
}
dict.total += 1
if ((dict.withCpCount > 0 && dict.total !== dict.withCpCount)
|| (dict.withoutCappingCount > 0 && dict.total !== dict.withoutCpCount)) {
throw new Error('All items should have the same shape')
}
}
if (dict.withoutCpCount) return false
console.timeEnd('checkList')
return true;
}
The function works well but I'm trying to generate some data using this function:
const generateList = (n) => {
return Array.from({length: n}, (x, i) => ({
lat: Number((Math.random() * 100).toFixed(5)),
lon: Number((Math.random() * 100).toFixed(5)),
rad: Number((Math.random() * 100).toFixed(5)),
cp: parseInt(Math.random() * 10000 + 1),
type: 'imp'
}))
}
Then I tested with huge value like 10Exp7 and 10Exp8 but I got a heap out of memory, and I wonder if I should increase the memory using export NODE_OPTIONS=--max-old-space-size=8192
or the function should be moree optimized.
I got this issue when I increase the number:
<--- Last few GCs --->
ca[12998:0x102d88000] 50019 ms: Mark-sweep 2047.7 (2050.4) -> 2046.7 (2050.4) MB, 741.7 / 0.0 ms (+ 8.2 ms in 17 steps since start of marking, biggest step 6.0 ms, walltime since start of marking 810 ms) (average mu = 0.165, current mu = 0.074) allocati[12998:0x102d88000] 51104 ms: Mark-sweep 2048.0 (2050.7) -> 2047.0 (2050.7) MB, 1011.9 / 0.0 ms (+ 12.6 ms in 17 steps since start of marking, biggest step 6.5 ms, walltime since start of marking 1085 ms) (average mu = 0.108, current mu = 0.056) alloc
<--- JS stacktrace --->
==== JS stack trace =========================================
0: ExitFrame [pc: 0x100a0d8d9]
Security context: 0x3506d33008d1 <JSObject>
1: /* anonymous */ [0x3506baaf7d59] [/Users/xxx/projects/utils/index.js:~59] [pc=0x3231b97054d9](this=0x3506d787fe99 <JSGlobal Object>,0x350675d804b1 <undefined>,21011726)
2: from [0x3506d331a659](this=0x3506d331a4b1 <JSFunction Array (sfi = 0x350622812af9)>,0x3506baaf7d91 <Object map = 0x350650b89949>,0x3506baaf7d59 <JSFunction (sfi = 0x3506618d4...
FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
1: 0x101204d65 node::Abort() (.cold.1) [/usr/local/bin/node]
2: 0x1000a5fd9 node::Abort() [/usr/local/bin/node]
3: 0x1000a613f node::OnFatalError(char const*, char const*) [/usr/local/bin/node]
4: 0x1001f0127 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/usr/local/bin/node]
xxxx@MacBook-Pro utils % node index.js
**************** Generate list *****************
<--- Last few GCs --->
[13738:0x102d88000] 33405 ms: Scavenge 2715.1 (2733.7) -> 2699.1 (2733.7) MB, 1.7 / 0.0 ms (average mu = 0.988, current mu = 0.988) allocation failure
[13738:0x102d88000] 33503 ms: Scavenge 2719.2 (2738.0) -> 2703.2 (2738.0) MB, 1.7 / 0.0 ms (average mu = 0.988, current mu = 0.988) allocation failure
[13738:0x102d88000] 33599 ms: Scavenge 2723.3 (2742.0) -> 2707.4 (2742.0) MB, 1.7 / 0.0 ms (average mu = 0.988, current mu = 0.988) allocation failure
<--- JS stacktrace --->
==== JS stack trace =========================================
0: ExitFrame [pc: 0x100a0d8d9]
Security context: 0x1362a92408d1 <JSObject>
1: from [0x1362a925a659](this=0x1362a925a4b1 <JSFunction Array (sfi = 0x1362567d2af9)>,0x1362a37045d9 <Object map = 0x13628ca099e9>,0x1362a37045a1 <JSFunction (sfi = 0x13621754b531)>)
2: generateList [0x1362a3704619] [/Users/xxx/projects/utils/index.js:59] [bytecode=0x13621754b5f1 offset=30](this=0x136275d3f759 <JSGlobal Object>,1000000000)
3:...
FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory
1: 0x101204d65 node::Abort() (.cold.1) [/usr/local/bin/node]
2: 0x1000a5fd9 node::Abort() [/usr/local/bin/node]
3: 0x1000a613f node::OnFatalError(char const*, char const*) [/usr/local/bin/node]
4: 0x1001f0127 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/usr/local/bin/node]
5: 0x1001f00c7 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/usr/local/bin/node]
6: 0x100388e85 v8::internal::Heap::FatalProcessOutOfMemory(char const*) [/usr/local/bin/node]
7: 0x10058df7f v8::internal::HashTable<v8::internal::NumberDictionary, v8::internal::NumberDictionaryShape>::EnsureCapacity(v8::internal::Isolate*, v8::internal::Handle<v8::internal::NumberDictionary>, int, v8::internal::AllocationType) [/usr/local/bin/node]
8: 0x10058eef2 v8::internal::Dictionary<v8::internal::NumberDictionary, v8::internal::NumberDictionaryShape>::Add(v8::internal::Isolate*, v8::internal::Handle<v8::internal::NumberDictionary>, unsigned int, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyDetails, int*) [/usr/local/bin/node]
9: 0x1004f6112 v8::internal::(anonymous namespace)::DictionaryElementsAccessor::AddImpl(v8::internal::Handle<v8::internal::JSObject>, unsigned int, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyAttributes, unsigned int) [/usr/local/bin/node]
10: 0x10055db56 v8::internal::JSObject::AddDataElement(v8::internal::Handle<v8::internal::JSObject>, unsigned int, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyAttributes) [/usr/local/bin/node]
11: 0x10059e71b v8::internal::Object::AddDataProperty(v8::internal::LookupIterator*, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyAttributes, v8::Maybe<v8::internal::ShouldThrow>, v8::internal::StoreOrigin) [/usr/local/bin/node]
12: 0x100551a08 v8::internal::JSObject::DefineOwnPropertyIgnoreAttributes(v8::internal::LookupIterator*, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyAttributes, v8::Maybe<v8::internal::ShouldThrow>, v8::internal::JSObject::AccessorInfoHandling) [/usr/local/bin/node]
13: 0x10054e074 v8::internal::JSObject::CreateDataProperty(v8::internal::LookupIterator*, v8::internal::Handle<v8::internal::Object>, v8::Maybe<v8::internal::ShouldThrow>) [/usr/local/bin/node]
14: 0x1006ccd10 v8::internal::Runtime_CreateDataProperty(int, unsigned long*, v8::internal::Isolate*) [/usr/local/bin/node]
15: 0x100a0d8d9 Builtins_CEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit [/usr/local/bin/node]
zsh: abort node index.js
It's just a question related to performance when iterating on a list with huge number of items.