I added a simple AABB broadphase to my physics system so i create/update arbiters / contacts only for body/shape pairs when there AABBs overlap.

How do i deal with arbiters which was not overlapping in that simulation step? Remember and remove after the broadphase - or is there a smarter way to do it?

Right now i am doing the simplest possible thing, which are very slow (One hashtable lookup for each body/shape pair + check shapes even when there was no overlap):

	//
	// Broad and narrow phase
	//
	// @SPEED: Real broadphase!
	BEGIN_BLOCK("Broadphase");
	for (u32 bodyIndexA = 0; bodyIndexA < physics->bodies.count; bodyIndexA++) {
		Body *bodyA = physics->bodies.used[bodyIndexA];
		for (u32 bodyIndexB = bodyIndexA + 1; bodyIndexB < physics->bodies.count; bodyIndexB++) {
			Body *bodyB = physics->bodies.used[bodyIndexB];
			if ((bodyA->type == BodyType::Dynamic) ||
				(bodyB->type == BodyType::Dynamic)) {
				b32 isOverlap = IsAABBOverlap(bodyA->aabb, bodyB->aabb);
				for (u32 shapeIndexA = 0; shapeIndexA < bodyA->shapeCount; ++shapeIndexA) {
					Shape *shapeA = bodyA->shapes + shapeIndexA;
					for (u32 shapeIndexB = 0; shapeIndexB < bodyB->shapeCount; ++shapeIndexB) {
						Shape *shapeB = bodyB->shapes + shapeIndexB;

						Arbiter tmpArbiter = {};
						InitArbiter(&tmpArbiter, bodyA, bodyB, shapeA, shapeB);

						// Initialize new arbiter on the stack
						if (isOverlap) {
							// Calculate transforms
							Transform worldTransformA = TransformMake(tmpArbiter.bodyA->position, tmpArbiter.bodyA->rotation);
							Transform worldTransformB = TransformMake(tmpArbiter.bodyB->position, tmpArbiter.bodyB->rotation);
							Transform transformA = TransformMult(tmpArbiter.shapeA->localTransform, worldTransformA);
							Transform transformB = TransformMult(tmpArbiter.shapeB->localTransform, worldTransformB);

							// Generate contacts for new arbiter
							tmpArbiter.contactCount = PhysicsContactsGenerate(physics, transformA, transformB, tmpArbiter.shapeA, tmpArbiter.shapeB, tmpArbiter.contacts);
						}

						// Get existing arbiter hash entry
#if PHYSICS_USE_STD_MAP
						HashTableEntry<Arbiter *> *oldArbiterEntry = nullptr;
						ArbiterKey arbiterKey = ArbiterKey(tmpArbiter.bodyA, tmpArbiter.bodyB, tmpArbiter.shapeA, tmpArbiter.shapeB);
						std::map<ArbiterKey, HashTableEntry<Arbiter *>>::iterator arbiterHashIterator = physics->arbiterHashMap.find(arbiterKey);
						if (arbiterHashIterator != physics->arbiterHashMap.end()) {
							oldArbiterEntry = &arbiterHashIterator->second;
						}
#else
						HashTableEntry<Arbiter *> *oldArbiterEntry = physics->arbiterHashMap.GetEntryWithCompare(tmpArbiter.hash, &tmpArbiter, CompareArbiters);
#endif
						// Update or remove arbiter and call enter / leave collision callbacks
						if (oldArbiterEntry != nullptr) {
							Arbiter *prevArbiter = oldArbiterEntry->value;
							if (tmpArbiter.contactCount > 0) {
								UpdateArbiter(physics, prevArbiter, &tmpArbiter);
							} else {
								if (physics->collisionLeave != nullptr) {
									physics->collisionLeave(oldArbiterEntry->value, physics->userData);
								}
								RemoveArbiter(physics, oldArbiterEntry);
							}
						} else if (tmpArbiter.contactCount > 0) {
							Arbiter *newArbiter = AddArbiter(physics, &tmpArbiter);
							if (physics->collisionEnter != nullptr) {
								physics->collisionEnter(newArbiter, physics->userData);
							}
						}
					}
				}
			}
		}
	}
	END_BLOCK();