Agfdhyk

I am trying to remove duplicate entry's from this json but it is only returning one object I am not understanding where I am going wrong.

The code is as follows.

// exemplary array of objects (id 'NewLive' occurs twice)

var arr = [

{"jobcodeid":{"S":"Etc_new"}},

{"jobcodeid":{"S":"NewLive"}},

{"jobcodeid":{"S":"NewLiveVid"}},

{"jobcodeid":{"S":"New_Live"}},

{"jobcodeid":{"S":"New_Live_Vid"}},

{"jobcodeid":{"S":"Newest"}},

{"jobcodeid":{"S":"NewestLive"}},

{"jobcodeid":{"S":"NewestLiveVid"}},

{"jobcodeid":{"S":"Very_New_Vid"}},

{"jobcodeid":{"S":"Etc_new"}},

{"jobcodeid":{"S":"NewLive"}},

{"jobcodeid":{"S":"NewLiveVid"}},

{"jobcodeid":{"S":"New_Live"}},

{"jobcodeid":{"S":"New_Live_Vid"}},

{"jobcodeid":{"S":"Newest"}},

{"jobcodeid":{"S":"NewestLive"}},

{"jobcodeid":{"S":"NewestLiveVid"}},

{"jobcodeid":{"S":"Very_New_Vid"}}

],

    obj = {}, new_arr = ;



// in the end the last unique object will be considered

arr.forEach(function(v){

    obj[v['id']] = v;

   console.log(JSON.stringify(new_arr));

});

new_arr = Object.keys(obj).map(function(id) { return obj[id]; });



console.log(JSON.stringify(new_arr));

I am attaching codepen also with this.

https://codepen.io/anon/pen/oQXJWK

Fist of all, you have to use obj[v['jobcodeid']] = v; instead of obj[v['id']] = v;.

but as v[jobcodeid] is an object, js will convert it to a string i.e. [object Object] and you will only one element in final array.

// exemplary array of objects (id 'NewLive' occurs twice)

var arr=[{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}},{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}}], obj = {}, new_arr = ;



// in the end the last unique object will be considered

arr.forEach(function(v){

    obj[v['jobcodeid']] = v;

});

new_arr = Object.keys(obj).map(function(id) { return obj[id]; });



console.log(JSON.stringify(new_arr));

You should use v.jobcodeid.S as keys of object.

// exemplary array of objects (id 'NewLive' occurs twice)

var arr=[{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}},{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}}], obj = {}, new_arr = ;



// in the end the last unique object will be considered

arr.forEach(function(v){

    obj[v.jobcodeid.S] = v;

});

new_arr = Object.keys(obj).map(function(id) { return obj[id]; });



console.log(JSON.stringify(new_arr));

The reason why your code is returning a single element is because you are using v['id'] but there is no id property on the objects, therefore throughout your loop you are setting obj[undefined] over and over.

In your jsfiddle code though this looks correct and the code seems to be working as intended.

In case someone got to this question to find out how to remove duplicates from an array in javascript, here are a few options:

The classic way: good old for loop

This is essentially the solution you used, iterate over the array, check if the key has been added to the result array, and if it's not there, add the element to the result.

Example:

const result = ;

const knownIDs = new Set();

for (const item of input) {

  if (!knownIDs.has(item.jobcodeid.S)) {

    result.push(item);

    knownIDs.add(item.jobcodeid.S);

  }

}

To Map and back

To filter duplicates you can convert the elements to a Map of key -> value, and then convert back into an array. This works because keys are unique in a Map, and duplicates will be automatically eliminated. The main advantage of this method is that due to the simplicity of the code it will have fewer bugs.

console.log(

  Array.from(

    new Map(

      input.map(i => [i.jobcodeid.S, i])

    ).values()

  )

)

filter and Set

Another option is to use a Set to record known ids and filter to remove items with known ids. The advantage of this method is that it might be easier to read since the intention is explicit. Also this is more performant than converting to Map and back.

const knownKeys = new Set();

console.log(

  input.filter(i => {

    if (!knownKeys.has(i.jobcodeid.S)) {

      knownKeys.add(i.jobcodeid.S);

      return true;

   }

  })

);

See them in action:

const input = [{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}},{"jobcodeid":{"S":"Etc_new"}},{"jobcodeid":{"S":"NewLive"}},{"jobcodeid":{"S":"NewLiveVid"}},{"jobcodeid":{"S":"New_Live"}},{"jobcodeid":{"S":"New_Live_Vid"}},{"jobcodeid":{"S":"Newest"}},{"jobcodeid":{"S":"NewestLive"}},{"jobcodeid":{"S":"NewestLiveVid"}},{"jobcodeid":{"S":"Very_New_Vid"}}];



// Classic for loop

const result = ;

const knownIDs = new Set();

for (const item of input) {

  if (!knownIDs.has(item.jobcodeid.S)) {

    result.push(item);

    knownIDs.add(item.jobcodeid.S);

  }

}



console.log(result.map(r => r.jobcodeid.S));



// To Map and back

console.log(

  Array.from(

    new Map(

      input.map(i => [i.jobcodeid.S, i])

    ).values()

  )

)



// filter and set

const knownKeys = new Set();

console.log(

  input.filter(i => {

    if (!knownKeys.has(i.jobcodeid.S)) {

      knownKeys.add(i.jobcodeid.S);

      return true;

   }

  })

);

For the record, I ran benchmarks on the accepted solution, mine and the performance improvements from Jacques' answer

accepted solution x 1,892,585 ops/sec ±3.48% (89 runs sampled)

Map and back x 495,116 ops/sec ±2.27% (90 runs sampled)

Set and filter x 1,600,833 ops/sec ±1.98% (90 runs sampled)

Jacques x 2,110,510 ops/sec ±0.98% (92 runs sampled)

Fastest is Jacques

As you can see, Jacques' solution is indeed twice as fast so if you are aiming to filter huge arrays or if performance is key, you should definitely choose that!

Posting an answer to show another way to do it with greater efficiency.

var arr = [

  {"jobcodeid":{"S":"Etc_new"}

  },

  {"jobcodeid":{"S":"NewLive"}

  },

{"jobcodeid":{"S":"NewLiveVid"}},

{"jobcodeid":{"S":"New_Live"}},

{"jobcodeid":{"S":"New_Live_Vid"}},

{"jobcodeid":{"S":"Newest"}},

{"jobcodeid":{"S":"NewestLive"}},

{"jobcodeid":{"S":"NewestLiveVid"}},

{"jobcodeid":{"S":"Very_New_Vid"}},

{"jobcodeid":{"S":"Etc_new"}},

{"jobcodeid":{"S":"NewLive"}},

{"jobcodeid":{"S":"NewLiveVid"}},

{"jobcodeid":{"S":"New_Live"}},

{"jobcodeid":{"S":"New_Live_Vid"}},

{"jobcodeid":{"S":"Newest"}},

{"jobcodeid":{"S":"NewestLive"}},

{"jobcodeid":{"S":"NewestLiveVid"}},

{"jobcodeid":{"S":"Very_New_Vid"}}

],

    obj = {}, new_arr = ;



// in the end the last unique object will be considered

for (const job of arr) {

	if (!obj[job.jobcodeid.S]) {

		obj[job.jobcodeid.S] = true;

		new_arr.push(job);

    }

}



console.log(JSON.stringify(new_arr));

This answer is always runs N iterations. When you loop through the keys after setting the unique values, it can run up to 2N iterations. (Changed from talking about Big O/Complexity to be more clear)

All you need is to use Set!

const arr = [

    { jobcodeid: { S: "Etc_new" } },

    { jobcodeid: { S: "NewLive" } },

    { jobcodeid: { S: "NewLiveVid" } },

    { jobcodeid: { S: "New_Live" } },

    { jobcodeid: { S: "New_Live_Vid" } },

    { jobcodeid: { S: "Newest" } },

    { jobcodeid: { S: "NewestLive" } },

    { jobcodeid: { S: "NewestLiveVid" } },

    { jobcodeid: { S: "Very_New_Vid" } },

    { jobcodeid: { S: "Etc_new" } },

    { jobcodeid: { S: "NewLive" } },

    { jobcodeid: { S: "NewLiveVid" } },

    { jobcodeid: { S: "New_Live" } },

    { jobcodeid: { S: "New_Live_Vid" } },

    { jobcodeid: { S: "Newest" } },

    { jobcodeid: { S: "NewestLive" } },

    { jobcodeid: { S: "NewestLiveVid" } },

    { jobcodeid: { S: "Very_New_Vid" } }

];



const uniqueItems = [...new Set(arr.map(i => i.jobcodeid.S))]