Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
# Changelog

## master / unreleased

* [CHANGE] Users Scanner: Rename user index update configuration. #7180
* Flag: Renamed `-*.users-scanner.user-index.cleanup-interval` to `-*.users-scanner.user-index.update-interval`.
* Config: Renamed `clean_up_interval` to `update_interval` within the `users_scanner` configuration block..
* [CHANGE] Querier: Refactored parquet cache configuration naming. #7146
* Metrics: Renamed `cortex_parquet_queryable_cache_*` to `cortex_parquet_cache_*`.
* Flags: Renamed `-querier.parquet-queryable-shard-cache-size` to `-querier.parquet-shard-cache-size` and `-querier.parquet-queryable-shard-cache-ttl` to `-querier.parquet-shard-cache-ttl`.
Expand Down
4 changes: 2 additions & 2 deletions docs/blocks-storage/querier.md
Original file line number Diff line number Diff line change
Expand Up @@ -1948,8 +1948,8 @@ blocks_storage:

# How frequently user index file is updated. It only takes effect when user
# scan strategy is user_index.
# CLI flag: -blocks-storage.users-scanner.user-index.cleanup-interval
[clean_up_interval: <duration> | default = 15m]
# CLI flag: -blocks-storage.users-scanner.user-index.update-interval
[update_interval: <duration> | default = 15m]

# TTL of the cached users. 0 disables caching and relies on caching at
# bucket client level.
Expand Down
4 changes: 2 additions & 2 deletions docs/blocks-storage/store-gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -2019,8 +2019,8 @@ blocks_storage:

# How frequently user index file is updated. It only takes effect when user
# scan strategy is user_index.
# CLI flag: -blocks-storage.users-scanner.user-index.cleanup-interval
[clean_up_interval: <duration> | default = 15m]
# CLI flag: -blocks-storage.users-scanner.user-index.update-interval
[update_interval: <duration> | default = 15m]

# TTL of the cached users. 0 disables caching and relies on caching at
# bucket client level.
Expand Down
12 changes: 6 additions & 6 deletions docs/configuration/config-file-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -999,8 +999,8 @@ users_scanner:

# How frequently user index file is updated. It only takes effect when user
# scan strategy is user_index.
# CLI flag: -alertmanager-storage.users-scanner.user-index.cleanup-interval
[clean_up_interval: <duration> | default = 15m]
# CLI flag: -alertmanager-storage.users-scanner.user-index.update-interval
[update_interval: <duration> | default = 15m]

# TTL of the cached users. 0 disables caching and relies on caching at bucket
# client level.
Expand Down Expand Up @@ -2624,8 +2624,8 @@ users_scanner:

# How frequently user index file is updated. It only takes effect when user
# scan strategy is user_index.
# CLI flag: -blocks-storage.users-scanner.user-index.cleanup-interval
[clean_up_interval: <duration> | default = 15m]
# CLI flag: -blocks-storage.users-scanner.user-index.update-interval
[update_interval: <duration> | default = 15m]

# TTL of the cached users. 0 disables caching and relies on caching at bucket
# client level.
Expand Down Expand Up @@ -5904,8 +5904,8 @@ users_scanner:

# How frequently user index file is updated. It only takes effect when user
# scan strategy is user_index.
# CLI flag: -ruler-storage.users-scanner.user-index.cleanup-interval
[clean_up_interval: <duration> | default = 15m]
# CLI flag: -ruler-storage.users-scanner.user-index.update-interval
[update_interval: <duration> | default = 15m]

# TTL of the cached users. 0 disables caching and relies on caching at bucket
# client level.
Expand Down
6 changes: 3 additions & 3 deletions integration/alertmanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@ func TestAlertmanagerWithUserIndexUpdater(t *testing.T) {

baseFlags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags())
flags := mergeFlags(baseFlags, AlertmanagerShardingFlags(consul.NetworkHTTPEndpoint(), 1), map[string]string{
"-alertmanager-storage.users-scanner.strategy": "user_index",
"-alertmanager-storage.users-scanner.user-index.cleanup-interval": "15s",
"-alertmanager.configs.poll-interval": "5s",
"-alertmanager-storage.users-scanner.strategy": "user_index",
"-alertmanager-storage.users-scanner.user-index.update-interval": "15s",
"-alertmanager.configs.poll-interval": "5s",
})

am := e2ecortex.NewAlertmanager(
Expand Down
8 changes: 4 additions & 4 deletions integration/ruler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,10 @@ func TestRulerWithUserIndexUpdater(t *testing.T) {
RulerFlags(),
RulerShardingFlags(consul.NetworkHTTPEndpoint()),
map[string]string{
"-ruler.sharding-strategy": "shuffle-sharding",
"-ruler-storage.users-scanner.strategy": "user_index",
"-ruler-storage.users-scanner.user-index.cleanup-interval": "15s",
"-ruler.tenant-shard-size": "1",
"-ruler.sharding-strategy": "shuffle-sharding",
"-ruler-storage.users-scanner.strategy": "user_index",
"-ruler-storage.users-scanner.user-index.update-interval": "15s",
"-ruler.tenant-shard-size": "1",
// Since we're not going to run any rule, we don't need the
// store-gateway to be configured to a valid address.
"-querier.store-gateway-addresses": "localhost:12345",
Expand Down
2 changes: 1 addition & 1 deletion pkg/alertmanager/alertstore/bucketclient/bucket_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func NewBucketAlertStore(bkt objstore.InstrumentedBucket, userScannerCfg users.U
baseScanner, _ := users.NewScanner(users.UsersScannerConfig{
Strategy: users.UserScanStrategyList,
}, alertBucket, logger, regWithComponent)
userIndexUpdater = users.NewUserIndexUpdater(alertBucket, userScannerCfg.CleanUpInterval, baseScanner, regWithComponent)
userIndexUpdater = users.NewUserIndexUpdater(alertBucket, userScannerCfg.UpdateInterval, baseScanner, regWithComponent)
}

return &BucketAlertStore{
Expand Down
4 changes: 2 additions & 2 deletions pkg/compactor/compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -762,7 +762,7 @@ func (c *Compactor) starting(ctx context.Context) error {
baseScanner, _ := users.NewScanner(users.UsersScannerConfig{
Strategy: users.UserScanStrategyList,
}, c.bucketClient, c.logger, c.registerer)
c.userIndexUpdater = users.NewUserIndexUpdater(c.bucketClient, c.storageCfg.UsersScanner.CleanUpInterval, baseScanner, extprom.WrapRegistererWith(prometheus.Labels{"component": "compactor"}, c.registerer))
c.userIndexUpdater = users.NewUserIndexUpdater(c.bucketClient, c.storageCfg.UsersScanner.UpdateInterval, baseScanner, extprom.WrapRegistererWith(prometheus.Labels{"component": "compactor"}, c.registerer))
}

return nil
Expand Down Expand Up @@ -1212,7 +1212,7 @@ func (c *Compactor) userIndexUpdateLoop(ctx context.Context) {
// Hardcode ID to check which compactor owns updating user index.
userID := users.UserIndexCompressedFilename
// Align with clean up interval.
ticker := time.NewTicker(util.DurationWithJitter(c.storageCfg.UsersScanner.CleanUpInterval, 0.1))
ticker := time.NewTicker(util.DurationWithJitter(c.storageCfg.UsersScanner.UpdateInterval, 0.1))
defer ticker.Stop()

for {
Expand Down
2 changes: 1 addition & 1 deletion pkg/compactor/compactor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1580,7 +1580,7 @@ func prepare(t *testing.T, compactorCfg Config, bucketClient objstore.Instrument
flagext.DefaultValues(&storageCfg)
storageCfg.BucketStore.BlockDiscoveryStrategy = string(cortex_tsdb.RecursiveDiscovery)
storageCfg.UsersScanner.Strategy = users.UserScanStrategyUserIndex
storageCfg.UsersScanner.CleanUpInterval = 100 * time.Millisecond // Short interval for testing
storageCfg.UsersScanner.UpdateInterval = 100 * time.Millisecond // Short interval for testing

// Create a temporary directory for compactor data.
compactorCfg.DataDir = t.TempDir()
Expand Down
2 changes: 1 addition & 1 deletion pkg/ruler/rulestore/bucketclient/bucket_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func NewBucketRuleStore(bkt objstore.Bucket, userScannerCfg users.UsersScannerCo
baseScanner, _ := users.NewScanner(users.UsersScannerConfig{
Strategy: users.UserScanStrategyList,
}, rulesBucket, logger, regWithComponent)
userIndexUpdater = users.NewUserIndexUpdater(rulesBucket, userScannerCfg.CleanUpInterval, baseScanner, regWithComponent)
userIndexUpdater = users.NewUserIndexUpdater(rulesBucket, userScannerCfg.UpdateInterval, baseScanner, regWithComponent)
}

return &BucketRuleStore{
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/users/index_updater_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func TestUserIndexUpdater_UpdateUserIndex(t *testing.T) {
t.Parallel()
bkt, _ := cortex_testutil.PrepareFilesystemBucket(t)

updater := NewUserIndexUpdater(bkt, defaultCleanUpInterval, testData.scanner, nil)
updater := NewUserIndexUpdater(bkt, defaultUpdateInterval, testData.scanner, nil)
err := updater.UpdateUserIndex(ctx)

if testData.expectErr {
Expand Down Expand Up @@ -118,7 +118,7 @@ func TestUserIndexUpdater_UpdateUserIndex_WriteError(t *testing.T) {
// Mock the bucket to return an error on upload
bkt.MockUpload(UserIndexCompressedFilename, assert.AnError)

updater := NewUserIndexUpdater(bkt, defaultCleanUpInterval, scanner, nil)
updater := NewUserIndexUpdater(bkt, defaultUpdateInterval, scanner, nil)
err := updater.UpdateUserIndex(ctx)
require.Error(t, err)
assert.Contains(t, err.Error(), "upload user index")
Expand Down
12 changes: 6 additions & 6 deletions pkg/util/users/scanner_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,17 @@ import (
)

type UsersScannerConfig struct {
Strategy string `yaml:"strategy"`
MaxStalePeriod time.Duration `yaml:"max_stale_period"`
CleanUpInterval time.Duration `yaml:"clean_up_interval"`
CacheTTL time.Duration `yaml:"cache_ttl"`
Strategy string `yaml:"strategy"`
MaxStalePeriod time.Duration `yaml:"max_stale_period"`
UpdateInterval time.Duration `yaml:"update_interval"`
CacheTTL time.Duration `yaml:"cache_ttl"`
}

const (
UserScanStrategyList = "list"
UserScanStrategyUserIndex = "user_index"

defaultCleanUpInterval = time.Minute * 15
defaultUpdateInterval = time.Minute * 15
)

var (
Expand All @@ -46,6 +46,6 @@ func (c *UsersScannerConfig) Validate() error {
func (c *UsersScannerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
f.StringVar(&c.Strategy, prefix+"users-scanner.strategy", UserScanStrategyList, fmt.Sprintf("Strategy to use to scan users. Supported values are: %s.", strings.Join(supportedStrategies, ", ")))
f.DurationVar(&c.MaxStalePeriod, prefix+"users-scanner.user-index.max-stale-period", time.Hour, "Maximum period of time to consider the user index as stale. Fall back to the base scanner if stale. Only valid when strategy is user_index.")
f.DurationVar(&c.CleanUpInterval, prefix+"users-scanner.user-index.cleanup-interval", defaultCleanUpInterval, fmt.Sprintf("How frequently user index file is updated. It only takes effect when user scan strategy is %s.", UserScanStrategyUserIndex))
f.DurationVar(&c.UpdateInterval, prefix+"users-scanner.user-index.update-interval", defaultUpdateInterval, fmt.Sprintf("How frequently user index file is updated. It only takes effect when user scan strategy is %s.", UserScanStrategyUserIndex))
f.DurationVar(&c.CacheTTL, prefix+"users-scanner.cache-ttl", 0, "TTL of the cached users. 0 disables caching and relies on caching at bucket client level.")
}
42 changes: 21 additions & 21 deletions schemas/cortex-config-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -912,13 +912,6 @@
"x-cli-flag": "alertmanager-storage.users-scanner.cache-ttl",
"x-format": "duration"
},
"clean_up_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "alertmanager-storage.users-scanner.user-index.cleanup-interval",
"x-format": "duration"
},
"max_stale_period": {
"default": "1h0m0s",
"description": "Maximum period of time to consider the user index as stale. Fall back to the base scanner if stale. Only valid when strategy is user_index.",
Expand All @@ -931,6 +924,13 @@
"description": "Strategy to use to scan users. Supported values are: list, user_index.",
"type": "string",
"x-cli-flag": "alertmanager-storage.users-scanner.strategy"
},
"update_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "alertmanager-storage.users-scanner.user-index.update-interval",
"x-format": "duration"
}
},
"type": "object"
Expand Down Expand Up @@ -3114,13 +3114,6 @@
"x-cli-flag": "blocks-storage.users-scanner.cache-ttl",
"x-format": "duration"
},
"clean_up_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "blocks-storage.users-scanner.user-index.cleanup-interval",
"x-format": "duration"
},
"max_stale_period": {
"default": "1h0m0s",
"description": "Maximum period of time to consider the user index as stale. Fall back to the base scanner if stale. Only valid when strategy is user_index.",
Expand All @@ -3133,6 +3126,13 @@
"description": "Strategy to use to scan users. Supported values are: list, user_index.",
"type": "string",
"x-cli-flag": "blocks-storage.users-scanner.strategy"
},
"update_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "blocks-storage.users-scanner.user-index.update-interval",
"x-format": "duration"
}
},
"type": "object"
Expand Down Expand Up @@ -7472,13 +7472,6 @@
"x-cli-flag": "ruler-storage.users-scanner.cache-ttl",
"x-format": "duration"
},
"clean_up_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "ruler-storage.users-scanner.user-index.cleanup-interval",
"x-format": "duration"
},
"max_stale_period": {
"default": "1h0m0s",
"description": "Maximum period of time to consider the user index as stale. Fall back to the base scanner if stale. Only valid when strategy is user_index.",
Expand All @@ -7491,6 +7484,13 @@
"description": "Strategy to use to scan users. Supported values are: list, user_index.",
"type": "string",
"x-cli-flag": "ruler-storage.users-scanner.strategy"
},
"update_interval": {
"default": "15m0s",
"description": "How frequently user index file is updated. It only takes effect when user scan strategy is user_index.",
"type": "string",
"x-cli-flag": "ruler-storage.users-scanner.user-index.update-interval",
"x-format": "duration"
}
},
"type": "object"
Expand Down
Loading