diff --git a/.aegir.js b/.aegir.js
index 0e01a44180..86fc7a30a5 100644
--- a/.aegir.js
+++ b/.aegir.js
@@ -48,7 +48,7 @@ const after = async () => {
 }
 
 module.exports = {
-  bundlesize: { maxSize: '215kB' },
+  bundlesize: { maxSize: '216kB' },
   hooks: {
     pre: before,
     post: after
diff --git a/doc/CONFIGURATION.md b/doc/CONFIGURATION.md
index 04993dc969..0376890b9d 100644
--- a/doc/CONFIGURATION.md
+++ b/doc/CONFIGURATION.md
@@ -23,6 +23,7 @@
       - [Setup with Auto Relay](#setup-with-auto-relay)
       - [Setup with Keychain](#setup-with-keychain)
       - [Configuring Dialing](#configuring-dialing)
+      - [Configuring Address Manager](#configuring-address-manager)
       - [Configuring Connection Manager](#configuring-connection-manager)
       - [Configuring Transport Manager](#configuring-transport-manager)
       - [Configuring Metrics](#configuring-metrics)
@@ -549,6 +550,26 @@ const node = await Libp2p.create({
   }
 ```
 
+#### Configuring Address Manager
+
+The address manager receives observed addresses from network peers. We accept observed addresses once a certain number of peers have reported the same observed address within a certain window of time.
+
+```js
+const node = await Libp2p.create({
+  addressManager: {
+    observedAddresses: {
+      // we must receive the same observed address from this many
+      // peers before we start believe it
+      minConfidence: 4,
+      // an address must reach the minimum level of confidence within
+      // this timeout otherwise it will be ignored
+      maxLifetimeBeforeEviction: (60 * 10) * 1000 // ten minutes in ms
+    }
+  },
+  // ...other options
+})
+```
+
 #### Configuring Connection Manager
 
 The Connection Manager prunes Connections in libp2p whenever certain limits are exceeded. If Metrics are enabled, you can also configure the Connection Manager to monitor the bandwidth of libp2p and prune connections as needed. You can read more about what Connection Manager does at [./CONNECTION_MANAGER.md](./CONNECTION_MANAGER.md). The configuration values below show the defaults for Connection Manager. See [./CONNECTION_MANAGER.md](./CONNECTION_MANAGER.md#options) for a full description of the parameters.
diff --git a/src/address-manager/index.js b/src/address-manager/index.js
index 625432252e..fabac08290 100644
--- a/src/address-manager/index.js
+++ b/src/address-manager/index.js
@@ -31,14 +31,18 @@ class AddressManager extends EventEmitter {
    * @param {object} [options]
    * @param {Array<string>} [options.listen = []] - list of multiaddrs string representation to listen.
    * @param {Array<string>} [options.announce = []] - list of multiaddrs string representation to announce.
+   * @param {object} [options.observedAddresses = { minConfidence: 4, maxLifetimeBeforeEviction: 600000 }] - configuration options for observed addresses
    */
-  constructor (peerId, { listen = [], announce = [] } = {}) {
+  constructor (peerId, { listen = [], announce = [], observedAddresses = { minConfidence: 4, maxLifetimeBeforeEviction: (60 * 10) * 1000 } } = {}) {
     super()
 
     this.peerId = peerId
     this.listen = new Set(listen.map(ma => ma.toString()))
     this.announce = new Set(announce.map(ma => ma.toString()))
-    this.observed = new Set()
+    this.observed = new Map()
+    this.config = {
+      observedAddresses
+    }
   }
 
   /**
@@ -65,15 +69,25 @@ class AddressManager extends EventEmitter {
    * @returns {Array<Multiaddr>}
    */
   getObservedAddrs () {
-    return Array.from(this.observed).map((a) => multiaddr(a))
+    const output = []
+
+    this.observed.forEach(({ confidence }, addr) => {
+      if (confidence >= this.config.observedAddresses.minConfidence) {
+        output.push(multiaddr(addr))
+      }
+    })
+
+    return output
   }
 
   /**
    * Add peer observed addresses
    *
    * @param {string | Multiaddr} addr
+   * @param {PeerId} reporter
+   * @param {number} [confidence=1]
    */
-  addObservedAddr (addr) {
+  addObservedAddr (addr, reporter, confidence = 1) {
     let ma = multiaddr(addr)
     const remotePeer = ma.getPeerId()
 
@@ -87,15 +101,41 @@ class AddressManager extends EventEmitter {
       }
     }
 
+    const now = Date.now()
     const addrString = ma.toString()
+    const wasNewAddr = !this.observed.has(addrString)
+    let addrRecord = {
+      confidence,
+      reporters: [
+        reporter.toB58String()
+      ],
+      expires: now + this.config.observedAddresses.maxLifetimeBeforeEviction
+    }
+
+    // we've seen this address before, increase the confidence we have in it
+    if (!wasNewAddr) {
+      addrRecord = this.observed.get(addrString)
 
-    // do not trigger the change:addresses event if we already know about this address
-    if (this.observed.has(addrString)) {
-      return
+      if (!addrRecord.reporters.includes(reporter.toB58String())) {
+        addrRecord.confidence++
+        addrRecord.reporters.push(reporter.toB58String())
+        addrRecord.expires = now + this.config.observedAddresses.maxLifetimeBeforeEviction
+      }
     }
 
-    this.observed.add(addrString)
-    this.emit('change:addresses')
+    this.observed.set(addrString, addrRecord)
+
+    // only emit event if we've reached the minimum confidence
+    if (addrRecord.confidence === this.config.observedAddresses.minConfidence) {
+      this.emit('change:addresses')
+    }
+
+    // evict addresses older than MAX_LOW_CONFIDENCE_ADDR_LIFETIME_MS we are not confident in
+    this.observed.forEach(({ confidence, expires }, key, map) => {
+      if (confidence < this.config.observedAddresses.minConfidence && expires < now) {
+        map.delete(key)
+      }
+    })
   }
 }
 
diff --git a/src/config.js b/src/config.js
index 6eefa42556..2b0a0677b7 100644
--- a/src/config.js
+++ b/src/config.js
@@ -16,6 +16,12 @@ const DefaultConfig = {
     announce: [],
     noAnnounce: []
   },
+  addressManager: {
+    observedAddresses: {
+      minConfidence: 4,
+      maxLifetimeBeforeEviction: (60 * 10) * 1000
+    }
+  },
   connectionManager: {
     minConnections: 25
   },
diff --git a/src/identify/index.js b/src/identify/index.js
index f5d8d4678d..8035a45d46 100644
--- a/src/identify/index.js
+++ b/src/identify/index.js
@@ -202,9 +202,8 @@ class IdentifyService {
     this.peerStore.protoBook.set(id, protocols)
     this.peerStore.metadataBook.set(id, 'AgentVersion', uint8ArrayFromString(message.agentVersion))
 
-    // TODO: Score our observed addr
     log('received observed address of %s', observedAddr)
-    this.addressManager.addObservedAddr(observedAddr)
+    this.addressManager.addObservedAddr(observedAddr, id)
   }
 
   /**
diff --git a/src/index.js b/src/index.js
index 034aaa5006..d9ba94f297 100644
--- a/src/index.js
+++ b/src/index.js
@@ -137,7 +137,10 @@ class Libp2p extends EventEmitter {
 
     // Addresses {listen, announce, noAnnounce}
     this.addresses = this._options.addresses
-    this.addressManager = new AddressManager(this.peerId, this._options.addresses)
+    this.addressManager = new AddressManager(this.peerId, {
+      ...this._options.addresses,
+      ...this._options.addressManager
+    })
 
     // when addresses change, update our peer record
     this.addressManager.on('change:addresses', () => {
diff --git a/src/nat-manager.js b/src/nat-manager.js
index 93c0251f81..7096c04801 100644
--- a/src/nat-manager.js
+++ b/src/nat-manager.js
@@ -16,11 +16,11 @@ const {
   codes: { ERR_INVALID_PARAMETERS }
 } = require('./errors')
 const isLoopback = require('libp2p-utils/src/multiaddr/is-loopback')
+const AddressManager = require('./address-manager')
 
 /**
  * @typedef {import('peer-id')} PeerId
  * @typedef {import('./transport-manager')} TransportManager
- * @typedef {import('./address-manager')} AddressManager
  */
 
 function highPort (min = 1024, max = 65535) {
@@ -118,11 +118,12 @@ class NatManager {
         protocol: transport.toUpperCase()
       })
 
+      // add with high confidence
       this._addressManager.addObservedAddr(Multiaddr.fromNodeAddress({
         family: 'IPv4',
         address: publicIp,
         port: `${publicPort}`
-      }, transport))
+      }, transport), this._peerId, this._addressManager.config.observedAddresses.minConfidence)
     }
   }
 
diff --git a/test/addresses/address-manager.spec.js b/test/addresses/address-manager.spec.js
index c27ef2992d..ea92b23258 100644
--- a/test/addresses/address-manager.spec.js
+++ b/test/addresses/address-manager.spec.js
@@ -15,9 +15,11 @@ const announceAddreses = ['/dns4/peer.io']
 
 describe('Address Manager', () => {
   let peerId
+  let peerIds
 
   before(async () => {
     peerId = await PeerId.createFromJSON(Peers[0])
+    peerIds = await Promise.all(Peers.slice(1).map(peerId => PeerId.createFromJSON(peerId)))
   })
 
   it('should not need any addresses', () => {
@@ -60,7 +62,7 @@ describe('Address Manager', () => {
 
     expect(am.observed).to.be.empty()
 
-    am.addObservedAddr('/ip4/123.123.123.123/tcp/39201')
+    am.addObservedAddr('/ip4/123.123.123.123/tcp/39201', peerId)
 
     expect(am.observed).to.have.property('size', 1)
   })
@@ -71,12 +73,12 @@ describe('Address Manager', () => {
 
     expect(am.observed).to.be.empty()
 
-    am.addObservedAddr(ma)
-    am.addObservedAddr(ma)
-    am.addObservedAddr(ma)
+    am.addObservedAddr(ma, peerId)
+    am.addObservedAddr(ma, peerId)
+    am.addObservedAddr(ma, peerId)
 
     expect(am.observed).to.have.property('size', 1)
-    expect(am.observed).to.include(ma)
+    expect(Array.from(am.observed.keys())).to.include(ma)
   })
 
   it('should only emit one change:addresses event', () => {
@@ -88,11 +90,25 @@ describe('Address Manager', () => {
       eventCount++
     })
 
-    am.addObservedAddr(ma)
-    am.addObservedAddr(ma)
-    am.addObservedAddr(ma)
-    am.addObservedAddr(`${ma}/p2p/${peerId}`)
-    am.addObservedAddr(`${ma}/p2p/${peerId.toB58String()}`)
+    am.addObservedAddr(ma, peerIds[0])
+    am.addObservedAddr(ma, peerIds[1])
+    am.addObservedAddr(ma, peerIds[2])
+    am.addObservedAddr(`${ma}/p2p/${peerId}`, peerIds[3])
+    am.addObservedAddr(`${ma}/p2p/${peerId.toB58String()}`, peerIds[4])
+
+    expect(eventCount).to.equal(1)
+  })
+
+  it('should emit one change:addresses event when specifying confidence', () => {
+    const ma = '/ip4/123.123.123.123/tcp/39201'
+    const am = new AddressManager(peerId)
+    let eventCount = 0
+
+    am.on('change:addresses', () => {
+      eventCount++
+    })
+
+    am.addObservedAddr(ma, peerId, am.config.observedAddresses.minConfidence)
 
     expect(eventCount).to.equal(1)
   })
@@ -103,11 +119,12 @@ describe('Address Manager', () => {
 
     expect(am.observed).to.be.empty()
 
-    am.addObservedAddr(ma)
-    am.addObservedAddr(`${ma}/p2p/${peerId}`)
+    am.addObservedAddr(ma, peerId)
+    am.addObservedAddr(`${ma}/p2p/${peerId}`, peerId)
 
     expect(am.observed).to.have.property('size', 1)
-    expect(am.observed).to.include(ma)
+
+    expect(Array.from(am.observed.keys())).to.include(ma)
   })
 
   it('should strip our peer address from added observed addresses in difference formats', () => {
@@ -116,12 +133,71 @@ describe('Address Manager', () => {
 
     expect(am.observed).to.be.empty()
 
-    am.addObservedAddr(ma)
-    am.addObservedAddr(`${ma}/p2p/${peerId}`) // base32 CID
-    am.addObservedAddr(`${ma}/p2p/${peerId.toB58String()}`) // base58btc
+    am.addObservedAddr(ma, peerId)
+    am.addObservedAddr(`${ma}/p2p/${peerId}`, peerId) // base32 CID
+    am.addObservedAddr(`${ma}/p2p/${peerId.toB58String()}`, peerId) // base58btc
 
     expect(am.observed).to.have.property('size', 1)
-    expect(am.observed).to.include(ma)
+
+    expect(Array.from(am.observed.keys())).to.include(ma)
+  })
+
+  it('should require a number of confirmations before believing address', () => {
+    const ma = '/ip4/123.123.123.123/tcp/39201'
+    const am = new AddressManager(peerId)
+
+    expect(am.observed).to.be.empty()
+
+    am.addObservedAddr(ma, peerId)
+
+    expect(am.getObservedAddrs().map(ma => ma.toString())).to.not.include(ma)
+
+    for (let i = 0; i < am.config.observedAddresses.minConfidence; i++) {
+      am.addObservedAddr(ma, peerIds[i])
+    }
+
+    expect(am.getObservedAddrs().map(ma => ma.toString())).to.include(ma)
+  })
+
+  it('should require a number of confirmations from different peers', () => {
+    const ma = '/ip4/123.123.123.123/tcp/39201'
+    const am = new AddressManager(peerId)
+
+    expect(am.observed).to.be.empty()
+
+    am.addObservedAddr(ma, peerId)
+
+    expect(am.getObservedAddrs().map(ma => ma.toString())).to.not.include(ma)
+
+    for (let i = 0; i < am.config.observedAddresses.minConfidence; i++) {
+      am.addObservedAddr(ma, peerIds[0])
+    }
+
+    expect(am.getObservedAddrs().map(ma => ma.toString())).to.not.include(ma)
+  })
+
+  it('should evict addresses that do not receive enough confirmations within the timeout', () => {
+    const ma1 = '/ip4/123.123.123.123/tcp/39201'
+    const ma2 = '/ip4/124.124.124.124/tcp/39202'
+    const am = new AddressManager(peerId)
+
+    expect(am.observed).to.be.empty()
+
+    am.addObservedAddr(ma1, peerId)
+
+    const observedAddrs = Array.from(am.observed.values())
+
+    expect(Array.from(am.observed.keys())).to.include(ma1)
+
+    // make expiry date a while ago
+    observedAddrs[0].expires = Date.now() - 1000
+
+    // will evict any old multiaddrs
+    am.addObservedAddr(ma2, peerId)
+
+    // should have been evicted
+    expect(Array.from(am.observed.keys())).to.not.include(ma1)
+    expect(Array.from(am.observed.keys())).to.include(ma2)
   })
 })
 
diff --git a/test/addresses/addresses.node.js b/test/addresses/addresses.node.js
index 719e52721a..c807e5d9da 100644
--- a/test/addresses/addresses.node.js
+++ b/test/addresses/addresses.node.js
@@ -164,7 +164,7 @@ describe('libp2p.multiaddrs', () => {
 
     expect(libp2p.multiaddrs).to.have.lengthOf(listenAddresses.length)
 
-    libp2p.addressManager.addObservedAddr(ma)
+    libp2p.addressManager.addObservedAddr(ma, libp2p.peerId, libp2p.addressManager.config.observedAddresses.minConfidence)
 
     expect(libp2p.multiaddrs).to.have.lengthOf(listenAddresses.length + 1)
     expect(libp2p.multiaddrs.map(ma => ma.toString())).to.include(ma)
diff --git a/test/core/consume-peer-record.spec.js b/test/core/consume-peer-record.spec.js
index c20a5efad6..3a39d463a9 100644
--- a/test/core/consume-peer-record.spec.js
+++ b/test/core/consume-peer-record.spec.js
@@ -37,7 +37,7 @@ describe('Consume peer record', () => {
       done = resolve
     })
 
-    libp2p.addressManager.addObservedAddr('/ip4/123.123.123.123/tcp/3983')
+    libp2p.addressManager.addObservedAddr('/ip4/123.123.123.123/tcp/3983', libp2p.peerId, libp2p.addressManager.config.observedAddresses.minConfidence)
 
     await p