Merge pull request #306 from RoaringBitmap/dlemire/remove_get_cardina…

…lity_buffer

Removing the getCardinality cache in run containers.

BitSliceIndexing/bsi.go

@@ -630,7 +630,7 @@ func (b *BSI) addDigit(foundSet *roaring.Bitmap, i int) {
 	}
 	carry := roaring.And(b.bA[i], foundSet)
 	b.bA[i].Xor(foundSet)
-	if carry.GetCardinality() > 0 {
+	if !carry.IsEmpty() {
 		if i+1 >= len(b.bA) {
 			b.bA = append(b.bA, roaring.NewBitmap())
 		}

README.md

@@ -384,12 +384,14 @@ You can help us test further the library with fuzzy testing:
          go get github.com/dvyukov/go-fuzz/go-fuzz-build
          go test -tags=gofuzz -run=TestGenerateSmatCorpus
          go-fuzz-build github.com/RoaringBitmap/roaring
-         go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+         go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat
 
 Let it run, and if the # of crashers is > 0, check out the reports in
 the workdir where you should be able to find the panic goroutine stack
 traces.
 
+You may also replace `-func FuzzSmat`  by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`.
+
 ### Alternative in Go
 
 There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring

arraycontainer.go

@@ -16,10 +16,11 @@ func (ac *arrayContainer) String() string {
 	return s + "}"
 }
 
-func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
 	for k := 0; k < len(ac.content); k++ {
 		x[k+i] = uint32(ac.content[k]) | mask
 	}
+	return i + len(ac.content)
 }
 
 func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
@@ -843,6 +844,10 @@ func (ac *arrayContainer) getCardinality() int {
 	return len(ac.content)
 }
 
+func (ac *arrayContainer) isEmpty() bool {
+	return len(ac.content) == 0
+}
+
 func (ac *arrayContainer) rank(x uint16) int {
 	answer := binarySearch(ac.content, x)
 	if answer >= 0 {
@@ -882,7 +887,7 @@ func (ac *arrayContainer) resetTo(a container) {
 		x.fillArray(ac.content)
 
 	case *runContainer16:
-		card := int(x.cardinality())
+		card := int(x.getCardinality())
 		ac.realloc(card)
 		cur := 0
 		for _, r := range x.iv {
@@ -956,10 +961,10 @@ func (ac *arrayContainer) numberOfRuns() (nr int) {
 				runlen++
 			} else {
 				if cur < prev {
-					panic("then fundamental arrayContainer assumption of sorted ac.content was broken")
+					panic("the fundamental arrayContainer assumption of sorted ac.content was broken")
 				}
 				if cur == prev {
-					panic("then fundamental arrayContainer assumption of deduplicated content was broken")
+					panic("the fundamental arrayContainer assumption of deduplicated content was broken")
 				} else {
 					nr++
 					runlen = 0

bitmapcontainer.go

@@ -264,7 +264,7 @@ func bitmapEquals(a, b []uint64) bool {
 	return true
 }
 
-func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
 	// TODO: should be written as optimized assembly
 	pos := i
 	base := mask
@@ -278,6 +278,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui
 		}
 		base += 64
 	}
+	return pos
 }
 
 func (bc *bitmapContainer) equals(o container) bool {
@@ -349,6 +350,11 @@ func (bc *bitmapContainer) getCardinality() int {
 	return bc.cardinality
 }
 
+
+func (bc *bitmapContainer) isEmpty() bool {
+	return bc.cardinality == 0
+}
+
 func (bc *bitmapContainer) clone() container {
 	ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))}
 	copy(ptr.bitmap, bc.bitmap[:])

bitmapcontainer_test.go

@@ -288,12 +288,12 @@ func TestBitmapContainerResetTo(t *testing.T) {
 	t.Run("to run container", func(t *testing.T) {
 		clean := newBitmapContainer()
 		clean.resetTo(run)
-		assert.EqualValues(t, clean.cardinality, run.cardinality())
+		assert.EqualValues(t, clean.cardinality, run.getCardinality())
 		assert.True(t, clean.toEfficientContainer().equals(run))
 
 		dirty := makeDirty()
 		dirty.resetTo(run)
-		assert.EqualValues(t, dirty.cardinality, run.cardinality())
+		assert.EqualValues(t, dirty.cardinality, run.getCardinality())
 		assert.True(t, dirty.toEfficientContainer().equals(run))
 	})
 }

fastaggregation.go

@@ -319,7 +319,7 @@ func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) {
 		}
 
 		result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored)
-		if result.getCardinality() > 0 {
+		if !result.isEmpty() {
 			x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false)
 			intersections++
 		}

go.mod

@@ -3,13 +3,16 @@ module github.com/RoaringBitmap/roaring
 go 1.14
 
 require (
+	github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72 // indirect
+	github.com/elazarl/go-bindata-assetfs v1.0.1 // indirect
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
 	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
 	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/stephens2424/writerset v1.0.2 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/willf/bitset v1.1.10
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect

go.sum

@@ -1,5 +1,10 @@
+github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72 h1:XiR1YwcWcRFzxjAhWK29HQL4nocj0QWJjpeRi/YASV0=
+github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
+github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
+github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
@@ -16,8 +21,12 @@ github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
+github.com/stephens2424/writerset v1.0.2 h1:znRLgU6g8RS5euYRcy004XeE4W+Tu44kALzy7ghPif8=
+github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
@@ -29,6 +38,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -46,6 +56,7 @@ golang.org/x/tools v0.0.0-20200928182047-19e03678916f h1:VwGa2Wf+rHGIxvsssCkUNIy
 golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

parallel.go

@@ -285,14 +285,14 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
 		for input := range inputChan {
 			c := input.containers[0].and(input.containers[1])
 			for _, next := range input.containers[2:] {
-				if c.getCardinality() == 0 {
+				if c.isEmpty() {
 					break
 				}
 				c = c.iand(next)
 			}
 
 			// Send a nil explicitly if the result of the intersection is an empty container
-			if c.getCardinality() == 0 {
+			if c.isEmpty() {
 				c = nil
 			}
 

roaring.go

@@ -151,8 +151,7 @@ func (rb *Bitmap) ToArray() []uint32 {
 		hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
 		c := rb.highlowcontainer.getContainerAtIndex(pos)
 		pos++
-		c.fillLeastSignificant16bits(array, pos2, hs)
-		pos2 += c.getCardinality()
+		pos2 = c.fillLeastSignificant16bits(array, pos2, hs)
 	}
 	return array
 }
@@ -542,7 +541,7 @@ func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) {
 			c := x.highlowcontainer.getContainerAtIndex(pos)
 			offsetted := c.addOffset(inOffset)
 
-			if offsetted[0].getCardinality() > 0 && (key >= 0 && key <= MaxUint16) {
+			if !offsetted[0].isEmpty() && (key >= 0 && key <= MaxUint16) {
 				curSize := answer.highlowcontainer.size()
 				lastkey := int32(0)
 
@@ -559,7 +558,7 @@ func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) {
 				}
 			}
 
-			if offsetted[1].getCardinality() > 0 && ((key+1) >= 0 && (key+1) <= MaxUint16) {
+			if !offsetted[1].isEmpty() && ((key+1) >= 0 && (key+1) <= MaxUint16) {
 				answer.highlowcontainer.appendContainer(uint16(key+1), offsetted[1], false)
 			}
 		}
@@ -630,13 +629,13 @@ func (rb *Bitmap) Remove(x uint32) {
 	if i >= 0 {
 		c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x))
 		rb.highlowcontainer.setContainerAtIndex(i, c)
-		if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
+		if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
 			rb.highlowcontainer.removeAtIndex(i)
 		}
 	}
 }
 
-// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present)
+// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively removed (and false if the integer was not present)
 func (rb *Bitmap) CheckedRemove(x uint32) bool {
 	// TODO: add unit tests for this method
 	hb := highbits(x)
@@ -646,7 +645,7 @@ func (rb *Bitmap) CheckedRemove(x uint32) bool {
 		oldcard := C.getCardinality()
 		C = C.iremoveReturnMinimized(lowbits(x))
 		rb.highlowcontainer.setContainerAtIndex(i, C)
-		if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
+		if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
 			rb.highlowcontainer.removeAtIndex(i)
 			return true
 		}
@@ -701,8 +700,9 @@ func (rb *Bitmap) Select(x uint32) (uint32, error) {
 	remaining := x
 	for i := 0; i < rb.highlowcontainer.size(); i++ {
 		c := rb.highlowcontainer.getContainerAtIndex(i)
-		if remaining >= uint32(c.getCardinality()) {
-			remaining -= uint32(c.getCardinality())
+		card := uint32(c.getCardinality())
+		if remaining >= card {
+			remaining -= card
 		} else {
 			key := rb.highlowcontainer.getKeyAtIndex(i)
 			return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil
@@ -729,7 +729,7 @@ main:
 					c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
 					c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
 					diff := c1.iand(c2)
-					if diff.getCardinality() > 0 {
+					if !diff.isEmpty() {
 						rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
 						intersectionsize++
 					}
@@ -931,7 +931,7 @@ func (rb *Bitmap) Xor(x2 *Bitmap) {
 			} else {
 				// TODO: couple be computed in-place for reduced memory usage
 				c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
-				if c.getCardinality() > 0 {
+				if !c.isEmpty() {
 					rb.highlowcontainer.setContainerAtIndex(pos1, c)
 					pos1++
 				} else {
@@ -1011,7 +1011,7 @@ main:
 					c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
 					c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
 					diff := c1.iandNot(c2)
-					if diff.getCardinality() > 0 {
+					if !diff.isEmpty() {
 						rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
 						intersectionsize++
 					}
@@ -1120,7 +1120,7 @@ main:
 				C := x1.highlowcontainer.getContainerAtIndex(pos1)
 				C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2))
 
-				if C.getCardinality() > 0 {
+				if !C.isEmpty() {
 					answer.highlowcontainer.appendContainer(s1, C, false)
 				}
 				pos1++
@@ -1167,7 +1167,7 @@ func Xor(x1, x2 *Bitmap) *Bitmap {
 				pos2++
 			} else {
 				c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
-				if c.getCardinality() > 0 {
+				if !c.isEmpty() {
 					answer.highlowcontainer.appendContainer(s1, c, false)
 				}
 				pos1++
@@ -1210,7 +1210,7 @@ main:
 					c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
 					c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
 					diff := c1.andNot(c2)
-					if diff.getCardinality() > 0 {
+					if !diff.isEmpty() {
 						answer.highlowcontainer.appendContainer(s1, diff, false)
 					}
 					pos1++
@@ -1300,7 +1300,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 
 		if i >= 0 {
 			c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
-			if c.getCardinality() > 0 {
+			if !c.isEmpty() {
 				rb.highlowcontainer.setContainerAtIndex(i, c)
 			} else {
 				rb.highlowcontainer.removeAtIndex(i)
@@ -1381,7 +1381,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
 			return
 		}
 		c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1))
-		if c.getCardinality() > 0 {
+		if !c.isEmpty() {
 			rb.highlowcontainer.setContainerAtIndex(i, c)
 		} else {
 			rb.highlowcontainer.removeAtIndex(i)
@@ -1394,7 +1394,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
 	if ifirst >= 0 {
 		if lbStart != 0 {
 			c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1))
-			if c.getCardinality() > 0 {
+			if !c.isEmpty() {
 				rb.highlowcontainer.setContainerAtIndex(ifirst, c)
 				ifirst++
 			}
@@ -1405,7 +1405,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
 	if ilast >= 0 {
 		if lbLast != max {
 			c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1))
-			if c.getCardinality() > 0 {
+			if !c.isEmpty() {
 				rb.highlowcontainer.setContainerAtIndex(ilast, c)
 			} else {
 				ilast++
@@ -1461,7 +1461,7 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
 
 		if i >= 0 {
 			c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
-			if c.getCardinality() > 0 {
+			if !c.isEmpty() {
 				answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
 			}
 

roaring64/bsi64.go

@@ -632,7 +632,7 @@ func (b *BSI) addDigit(foundSet *Bitmap, i int) {
 	}
 	carry := And(b.bA[i], foundSet)
 	b.bA[i].Xor(foundSet)
-	if carry.GetCardinality() > 0 {
+	if !carry.IsEmpty() {
 		if i+1 >= len(b.bA) {
 			b.bA = append(b.bA, NewBitmap())
 		}

roaringarray.go

@@ -17,6 +17,7 @@ type container interface {
 	iand(container) container // i stands for inplace
 	andNot(container) container
 	iandNot(container) container // i stands for inplace
+	isEmpty() bool
 	getCardinality() int
 	// rank returns the number of integers that are
 	// smaller or equal to x. rank(infinity) would be getCardinality().
@@ -47,7 +48,7 @@ type container interface {
 	// any of the implementations.
 	equals(r container) bool
 
-	fillLeastSignificant16bits(array []uint32, i int, mask uint32)
+	fillLeastSignificant16bits(array []uint32, i int, mask uint32) int
 	or(r container) container
 	orCardinality(r container) int
 	isFull() bool
@@ -645,7 +646,6 @@ func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte
 
 			nb := runContainer16{
 				iv:   byteSliceAsInterval16Slice(buf),
-				card: int64(card),
 			}
 
 			ra.containers[i] = &nb