@@ -44,8 +44,6 @@ CacheAllocator<CacheTrait>::CacheAllocator(Config config)
44
44
[this ](Item* it) -> ItemHandle { return acquire (it); })),
45
45
chainedItemLocks_ (config_.chainedItemsLockPower,
46
46
std::make_shared<MurmurHash2>()),
47
- movesMap_(kShards ),
48
- moveLock_(kShards ),
49
47
cacheCreationTime_{util::getCurrentTimeSec ()} {
50
48
51
49
if (numTiers_ > 1 || std::holds_alternative<FileShmSegmentOpts>(
@@ -132,8 +130,6 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
132
130
[this](Item* it) -> ItemHandle { return acquire (it); })),
133
131
chainedItemLocks_(config_.chainedItemsLockPower,
134
132
std::make_shared<MurmurHash2>()),
135
- movesMap_(kShards ),
136
- moveLock_(kShards ),
137
133
cacheCreationTime_{util::getCurrentTimeSec ()} {
138
134
initCommon (false );
139
135
shmManager_->removeShm (detail::kShmInfoName ,
@@ -170,8 +166,6 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
170
166
[this](Item* it) -> ItemHandle { return acquire (it); })),
171
167
chainedItemLocks_(config_.chainedItemsLockPower,
172
168
std::make_shared<MurmurHash2>()),
173
- movesMap_(kShards ),
174
- moveLock_(kShards ),
175
169
cacheCreationTime_{*metadata_.cacheCreationTime_ref ()} {
176
170
/* TODO - per tier? */
177
171
for (auto pid : *metadata_.compactCachePools_ref ()) {
@@ -272,6 +266,14 @@ void CacheAllocator<CacheTrait>::initCommon(bool dramCacheAttached) {
272
266
nvmAdmissionPolicy_->initMinTTL (config_.nvmAdmissionMinTTL );
273
267
}
274
268
}
269
+ if (numTiers_ > 1 && !config_.moveCb ) {
270
+ XLOG (WARN, " No moveCb set, using memcpy for moving items between tiers." );
271
+ config_.moveCb = [](Item& oldItem, Item& newItem, Item* parentItem){
272
+ if (parentItem != nullptr )
273
+ throw std::runtime_error (" TODO: chained items not supported" );
274
+ std::memcpy (newItem.getMemory (), oldItem.getMemory (), oldItem.getSize ());
275
+ };
276
+ }
275
277
initStats ();
276
278
initNvmCache (dramCacheAttached);
277
279
initWorkers ();
@@ -1289,159 +1291,11 @@ CacheAllocator<CacheTrait>::insertOrReplace(const ItemHandle& handle) {
1289
1291
return replaced;
1290
1292
}
1291
1293
1292
- /* Next two methods are used to asynchronously move Item between memory tiers.
1293
- *
1294
- * The thread, which moves Item, allocates new Item in the tier we are moving to
1295
- * and calls moveRegularItemOnEviction() method. This method does the following:
1296
- * 1. Create MoveCtx and put it to the movesMap.
1297
- * 2. Update the access container with the new item from the tier we are
1298
- * moving to. This Item has kIncomplete flag set.
1299
- * 3. Copy data from the old Item to the new one.
1300
- * 4. Unset the kIncomplete flag and Notify MoveCtx
1301
- *
1302
- * Concurrent threads which are getting handle to the same key:
1303
- * 1. When a handle is created it checks if the kIncomplete flag is set
1304
- * 2. If so, Handle implementation creates waitContext and adds it to the
1305
- * MoveCtx by calling addWaitContextForMovingItem() method.
1306
- * 3. Wait until the moving thread will complete its job.
1307
- */
1308
- template <typename CacheTrait>
1309
- bool CacheAllocator<CacheTrait>::addWaitContextForMovingItem(
1310
- folly::StringPiece key, std::shared_ptr<WaitContext<ReadHandle>> waiter) {
1311
- auto shard = getShardForKey (key);
1312
- auto & movesMap = getMoveMapForShard (shard);
1313
- auto lock = getMoveLockForShard (shard);
1314
- auto it = movesMap.find (key);
1315
- if (it == movesMap.end ()) {
1316
- return false ;
1317
- }
1318
- auto ctx = it->second .get ();
1319
- ctx->addWaiter (std::move (waiter));
1320
- return true ;
1321
- }
1322
-
1323
- template <typename CacheTrait>
1324
- typename CacheAllocator<CacheTrait>::ItemHandle
1325
- CacheAllocator<CacheTrait>::moveRegularItemOnEviction(
1326
- Item& oldItem, ItemHandle& newItemHdl) {
1327
- XDCHECK (oldItem.isMoving ());
1328
- // TODO: should we introduce new latency tracker. E.g. evictRegularLatency_
1329
- // ??? util::LatencyTracker tracker{stats_.evictRegularLatency_};
1330
-
1331
- if (!oldItem.isAccessible () || oldItem.isExpired ()) {
1332
- return {};
1333
- }
1334
-
1335
- XDCHECK_EQ (newItemHdl->getSize (), oldItem.getSize ());
1336
- XDCHECK_NE (getTierId (oldItem), getTierId (*newItemHdl));
1337
-
1338
- // take care of the flags before we expose the item to be accessed. this
1339
- // will ensure that when another thread removes the item from RAM, we issue
1340
- // a delete accordingly. See D7859775 for an example
1341
- if (oldItem.isNvmClean ()) {
1342
- newItemHdl->markNvmClean ();
1343
- }
1344
-
1345
- folly::StringPiece key (oldItem.getKey ());
1346
- auto shard = getShardForKey (key);
1347
- auto & movesMap = getMoveMapForShard (shard);
1348
- MoveCtx* ctx (nullptr );
1349
- {
1350
- auto lock = getMoveLockForShard (shard);
1351
- auto res = movesMap.try_emplace (key, std::make_unique<MoveCtx>());
1352
- if (!res.second ) {
1353
- return {};
1354
- }
1355
- ctx = res.first ->second .get ();
1356
- }
1357
-
1358
- auto resHdl = ItemHandle{};
1359
- auto guard = folly::makeGuard ([key, this , ctx, shard, &resHdl]() {
1360
- auto & movesMap = getMoveMapForShard (shard);
1361
- if (resHdl)
1362
- resHdl->unmarkIncomplete ();
1363
- auto lock = getMoveLockForShard (shard);
1364
- ctx->setItemHandle (std::move (resHdl));
1365
- movesMap.erase (key);
1366
- });
1367
-
1368
- // TODO: Possibly we can use markMoving() instead. But today
1369
- // moveOnSlabRelease logic assume that we mark as moving old Item
1370
- // and than do copy and replace old Item with the new one in access
1371
- // container. Furthermore, Item can be marked as Moving only
1372
- // if it is linked to MM container. In our case we mark the new Item
1373
- // and update access container before the new Item is ready (content is
1374
- // copied).
1375
- newItemHdl->markIncomplete ();
1376
-
1377
- // Inside the access container's lock, this checks if the old item is
1378
- // accessible and its refcount is zero. If the item is not accessible,
1379
- // there is no point to replace it since it had already been removed
1380
- // or in the process of being removed. If the item is in cache but the
1381
- // refcount is non-zero, it means user could be attempting to remove
1382
- // this item through an API such as remove(ItemHandle). In this case,
1383
- // it is unsafe to replace the old item with a new one, so we should
1384
- // also abort.
1385
- if (!accessContainer_->replaceIf (oldItem, *newItemHdl,
1386
- itemMovingPredicate)) {
1387
- return {};
1388
- }
1389
-
1390
- if (config_.moveCb ) {
1391
- // Execute the move callback. We cannot make any guarantees about the
1392
- // consistency of the old item beyond this point, because the callback can
1393
- // do more than a simple memcpy() e.g. update external references. If there
1394
- // are any remaining handles to the old item, it is the caller's
1395
- // responsibility to invalidate them. The move can only fail after this
1396
- // statement if the old item has been removed or replaced, in which case it
1397
- // should be fine for it to be left in an inconsistent state.
1398
- config_.moveCb (oldItem, *newItemHdl, nullptr );
1399
- } else {
1400
- std::memcpy (newItemHdl->getWritableMemory (), oldItem.getMemory (),
1401
- oldItem.getSize ());
1402
- }
1403
-
1404
- // Inside the MM container's lock, this checks if the old item exists to
1405
- // make sure that no other thread removed it, and only then replaces it.
1406
- if (!replaceInMMContainer (oldItem, *newItemHdl)) {
1407
- accessContainer_->remove (*newItemHdl);
1408
- return {};
1409
- }
1410
-
1411
- // Replacing into the MM container was successful, but someone could have
1412
- // called insertOrReplace() or remove() before or after the
1413
- // replaceInMMContainer() operation, which would invalidate newItemHdl.
1414
- if (!newItemHdl->isAccessible ()) {
1415
- removeFromMMContainer (*newItemHdl);
1416
- return {};
1417
- }
1418
-
1419
- // no one can add or remove chained items at this point
1420
- if (oldItem.hasChainedItem ()) {
1421
- // safe to acquire handle for a moving Item
1422
- auto oldHandle = acquire (&oldItem);
1423
- XDCHECK_EQ (1u , oldHandle->getRefCount ()) << oldHandle->toString ();
1424
- XDCHECK (!newItemHdl->hasChainedItem ()) << newItemHdl->toString ();
1425
- try {
1426
- auto l = chainedItemLocks_.lockExclusive (oldItem.getKey ());
1427
- transferChainLocked (oldHandle, newItemHdl);
1428
- } catch (const std::exception& e) {
1429
- // this should never happen because we drained all the handles.
1430
- XLOGF (DFATAL, " {}" , e.what ());
1431
- throw ;
1432
- }
1433
-
1434
- XDCHECK (!oldItem.hasChainedItem ());
1435
- XDCHECK (newItemHdl->hasChainedItem ());
1436
- }
1437
- newItemHdl.unmarkNascent ();
1438
- resHdl = std::move (newItemHdl); // guard will assign it to ctx under lock
1439
- return acquire (&oldItem);
1440
- }
1441
-
1442
1294
template <typename CacheTrait>
1295
+ template <typename Predicate>
1443
1296
bool CacheAllocator<CacheTrait>::moveRegularItem(Item& oldItem,
1444
- ItemHandle& newItemHdl) {
1297
+ ItemHandle& newItemHdl,
1298
+ Predicate &&predicate) {
1445
1299
XDCHECK (config_.moveCb );
1446
1300
util::LatencyTracker tracker{stats_.moveRegularLatency_ };
1447
1301
@@ -1450,8 +1304,6 @@ bool CacheAllocator<CacheTrait>::moveRegularItem(Item& oldItem,
1450
1304
}
1451
1305
1452
1306
XDCHECK_EQ (newItemHdl->getSize (), oldItem.getSize ());
1453
- XDCHECK_EQ (reinterpret_cast <uintptr_t >(&getMMContainer (oldItem)),
1454
- reinterpret_cast <uintptr_t >(&getMMContainer (*newItemHdl)));
1455
1307
1456
1308
// take care of the flags before we expose the item to be accessed. this
1457
1309
// will ensure that when another thread removes the item from RAM, we issue
@@ -1477,7 +1329,7 @@ bool CacheAllocator<CacheTrait>::moveRegularItem(Item& oldItem,
1477
1329
// this item through an API such as remove(ItemHandle). In this case,
1478
1330
// it is unsafe to replace the old item with a new one, so we should
1479
1331
// also abort.
1480
- if (!accessContainer_->replaceIf (oldItem, *newItemHdl, itemMovingPredicate )) {
1332
+ if (!accessContainer_->replaceIf (oldItem, *newItemHdl, predicate )) {
1481
1333
return false ;
1482
1334
}
1483
1335
@@ -1518,63 +1370,6 @@ bool CacheAllocator<CacheTrait>::moveRegularItem(Item& oldItem,
1518
1370
return true ;
1519
1371
}
1520
1372
1521
- template <typename CacheTrait>
1522
- bool CacheAllocator<CacheTrait>::moveRegularItemForPromotion(Item& oldItem,
1523
- ItemHandle& newItemHdl) {
1524
- XDCHECK (config_.moveCb );
1525
- util::LatencyTracker tracker{stats_.moveRegularLatency_ };
1526
-
1527
- if (!oldItem.isAccessible () || oldItem.isExpired ()) {
1528
- return false ;
1529
- }
1530
-
1531
- XDCHECK_EQ (newItemHdl->getSize (), oldItem.getSize ());
1532
-
1533
- if (config_.moveCb ) {
1534
- // Execute the move callback. We cannot make any guarantees about the
1535
- // consistency of the old item beyond this point, because the callback can
1536
- // do more than a simple memcpy() e.g. update external references. If there
1537
- // are any remaining handles to the old item, it is the caller's
1538
- // responsibility to invalidate them. The move can only fail after this
1539
- // statement if the old item has been removed or replaced, in which case it
1540
- // should be fine for it to be left in an inconsistent state.
1541
- config_.moveCb (oldItem, *newItemHdl, nullptr );
1542
- } else {
1543
- std::memcpy (newItemHdl->getWritableMemory (), oldItem.getMemory (),
1544
- oldItem.getSize ());
1545
- }
1546
-
1547
- auto predicate = [this ](const Item& item) {
1548
- // if inclusive cache is allowed, replace even if there are active users.
1549
- return config_.numDuplicateElements > 0 || item.getRefCount () == 0 ;
1550
- };
1551
- if (!accessContainer_->replaceIf (oldItem, *newItemHdl, predicate)) {
1552
- return false ;
1553
- }
1554
-
1555
- // Inside the MM container's lock, this checks if the old item exists to
1556
- // make sure that no other thread removed it, and only then replaces it.
1557
- if (!replaceInMMContainer (oldItem, *newItemHdl)) {
1558
- accessContainer_->remove (*newItemHdl);
1559
- return false ;
1560
- }
1561
-
1562
- // Replacing into the MM container was successful, but someone could have
1563
- // called insertOrReplace() or remove() before or after the
1564
- // replaceInMMContainer() operation, which would invalidate newItemHdl.
1565
- if (!newItemHdl->isAccessible ()) {
1566
- removeFromMMContainer (*newItemHdl);
1567
- return false ;
1568
- }
1569
-
1570
- // no one can add or remove chained items at this point
1571
- if (oldItem.hasChainedItem ()) {
1572
- throw std::runtime_error (" Not supported" );
1573
- }
1574
- newItemHdl.unmarkNascent ();
1575
- return true ;
1576
- }
1577
-
1578
1373
template <typename CacheTrait>
1579
1374
bool CacheAllocator<CacheTrait>::moveChainedItem(ChainedItem& oldItem,
1580
1375
ItemHandle& newItemHdl) {
@@ -1814,8 +1609,9 @@ CacheAllocator<CacheTrait>::tryEvictToNextMemoryTier(
1814
1609
1815
1610
if (newItemHdl) {
1816
1611
XDCHECK_EQ (newItemHdl->getSize (), item.getSize ());
1817
-
1818
- return moveRegularItemOnEviction (item, newItemHdl);
1612
+ if (tryMovingForSlabRelease (item, newItemHdl, itemMovingPredicate)) {
1613
+ return acquire (&item);
1614
+ }
1819
1615
}
1820
1616
}
1821
1617
@@ -1841,7 +1637,11 @@ CacheAllocator<CacheTrait>::tryPromoteToNextMemoryTier(
1841
1637
1842
1638
if (newItemHdl) {
1843
1639
XDCHECK_EQ (newItemHdl->getSize (), item.getSize ());
1844
- if (moveRegularItemForPromotion (item, newItemHdl)) {
1640
+ auto predicate = [this ](const Item& item) {
1641
+ // if inclusive cache is allowed, replace even if there are active users.
1642
+ return config_.numDuplicateElements > 0 || item.getRefCount () == 0 ;
1643
+ };
1644
+ if (tryMovingForSlabRelease (item, newItemHdl, std::move (predicate))) {
1845
1645
return true ;
1846
1646
}
1847
1647
}
@@ -2918,7 +2718,7 @@ bool CacheAllocator<CacheTrait>::moveForSlabRelease(
2918
2718
2919
2719
// if we have a valid handle, try to move, if not, we retry.
2920
2720
if (newItemHdl) {
2921
- isMoved = tryMovingForSlabRelease (oldItem, newItemHdl);
2721
+ isMoved = tryMovingForSlabRelease (oldItem, newItemHdl, itemMovingPredicate );
2922
2722
if (isMoved) {
2923
2723
break ;
2924
2724
}
@@ -3041,8 +2841,9 @@ CacheAllocator<CacheTrait>::allocateNewItemForOldItem(const Item& oldItem) {
3041
2841
}
3042
2842
3043
2843
template <typename CacheTrait>
2844
+ template <typename Predicate>
3044
2845
bool CacheAllocator<CacheTrait>::tryMovingForSlabRelease(
3045
- Item& oldItem, ItemHandle& newItemHdl) {
2846
+ Item& oldItem, ItemHandle& newItemHdl, Predicate&& predicate ) {
3046
2847
// By holding onto a user-level synchronization object, we ensure moving
3047
2848
// a regular item or chained item is synchronized with any potential
3048
2849
// user-side mutation.
@@ -3074,7 +2875,7 @@ bool CacheAllocator<CacheTrait>::tryMovingForSlabRelease(
3074
2875
3075
2876
return oldItem.isChainedItem ()
3076
2877
? moveChainedItem (oldItem.asChainedItem (), newItemHdl)
3077
- : moveRegularItem (oldItem, newItemHdl);
2878
+ : moveRegularItem (oldItem, newItemHdl, std::move (predicate) );
3078
2879
}
3079
2880
3080
2881
template <typename CacheTrait>
0 commit comments