mirror of https://github.com/mongodb/mongo
SERVER-67057 create scons cache metrics
This commit is contained in:
Richard Samuels
Evergreen Agent
parent
1a623ce0a0
commit
aa6dbc7ca9
|
|
@ -36,6 +36,7 @@ from .memory import MemoryMonitor
|
|||
from .per_action_metrics import PerActionMetrics
|
||||
from .artifacts import CollectArtifacts
|
||||
from .scons import SConsStats
|
||||
from .cache_dir import CacheDirCollector, CacheDirValidateWithMetrics
|
||||
|
||||
_SEC_TO_NANOSEC_FACTOR = 1000000000.0
|
||||
_METRICS_COLLECTORS = []
|
||||
|
|
@ -88,9 +89,12 @@ def generate(env, **kwargs):
|
|||
MemoryMonitor(psutil.Process().memory_info().vms),
|
||||
PerActionMetrics(),
|
||||
CollectArtifacts(env),
|
||||
SConsStats()
|
||||
SConsStats(),
|
||||
CacheDirCollector()
|
||||
]
|
||||
|
||||
env['CACHEDIR_CLASS'] = CacheDirValidateWithMetrics
|
||||
|
||||
|
||||
def exists(env):
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -13,16 +13,14 @@
|
|||
"description": "Size in bytes",
|
||||
"minimum": 0
|
||||
},
|
||||
"seconds": {
|
||||
"type": "integer",
|
||||
"description": "Number of seconds"
|
||||
},
|
||||
"binsize": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filesize": { "$ref": "#/$defs/bytes" },
|
||||
"vmsize": { "$ref": "#/$defs/bytes" }
|
||||
}
|
||||
},
|
||||
"required": ["filesize", "vmsize"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"type" : "object",
|
||||
|
|
@ -48,7 +46,9 @@
|
|||
"max": { "$ref": "#/$defs/memory" },
|
||||
"arithmetic_mean": {"type": "number"},
|
||||
"start_mem": { "$ref": "#/$defs/memory" }
|
||||
}
|
||||
},
|
||||
"required": ["mem_over_time", "max", "arithmetic_mean", "start_mem"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"artifact_metrics": {
|
||||
"type": "object",
|
||||
|
|
@ -83,66 +83,99 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"build_tasks" : {
|
||||
"type": "array",
|
||||
"task": {
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"array_index": {"type": "integer"},
|
||||
"start_time": {"$ref": "#/$defs/timestamp"},
|
||||
"end_time" : {"$ref": "#/$defs/timestamp"},
|
||||
"cpu_time" : {"type" : "integer"},
|
||||
"builder" : {"type" : "str"},
|
||||
"builder" : {"type" : "string"},
|
||||
"mem_usage": {"$ref": "#/$defs/memory"},
|
||||
"inputs" : {
|
||||
"type": "array",
|
||||
"input": { "type": "str"}
|
||||
"items": { "type": "string"}
|
||||
},
|
||||
"outputs" : {
|
||||
"type": "array",
|
||||
"output": { "type": "str"}
|
||||
"items": { "type": "string"}
|
||||
},
|
||||
"action": {"type" : "str"}
|
||||
}
|
||||
"action": {"type" : "string"}
|
||||
},
|
||||
"required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"scons_metrics": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"memory": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pre_read": {"$ref": "#/$defs/bytes"},
|
||||
"post_read": {"$ref": "#/$defs/bytes"},
|
||||
"pre_build": {"$ref": "#/$defs/bytes"},
|
||||
"post_build": {"$ref": "#/$defs/bytes"}
|
||||
}
|
||||
},
|
||||
"cache_metrics": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cache_artifacts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"array_index": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
"size": {"$ref": "#/$defs/bytes"}
|
||||
},
|
||||
"required": ["array_index", "name", "size"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"push_time": { "$ref": "#/$defs/timestamp" },
|
||||
"pull_time": { "$ref": "#/$defs/timestamp" },
|
||||
"cache_size": { "$ref": "#/$defs/bytes" }
|
||||
},
|
||||
"time": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"total": {"type": "number"},
|
||||
"sconscript_exec": {"type": "number"},
|
||||
"scons_exec": {"type": "number"},
|
||||
"command_exec": {"type": "number"}
|
||||
}
|
||||
},
|
||||
"counts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"required": ["cache_artifacts", "push_time", "pull_time", "cache_size"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"scons_metrics": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"memory": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"array_index": {"type": "integer"},
|
||||
"item_name": {"type": "str"},
|
||||
"pre_read": {"$ref": "#/$defs/bytes"},
|
||||
"post_read": {"$ref": "#/$defs/bytes"},
|
||||
"pre_build": {"$ref": "#/$defs/bytes"},
|
||||
"post_build": {"$ref": "#/$defs/bytes"}
|
||||
},
|
||||
"required": ["pre_read", "post_read", "pre_build", "post_build"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"time": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"total": {"type": "number"},
|
||||
"sconscript_exec": {"type": "number"},
|
||||
"scons_exec": {"type": "number"},
|
||||
"command_exec": {"type": "number"}
|
||||
},
|
||||
"required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"counts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"array_index": {"type": "integer"},
|
||||
"item_name": {"type": "string"},
|
||||
"pre_read": {"$ref": "#/$defs/bytes"},
|
||||
"post_read": {"$ref": "#/$defs/bytes"},
|
||||
"pre_build": {"$ref": "#/$defs/bytes"},
|
||||
"post_build": {"$ref": "#/$defs/bytes"}
|
||||
},
|
||||
"required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
import os
|
||||
import time
|
||||
from site_tools.validate_cache_dir import CacheDirValidate
|
||||
from .protocol import BuildMetricsCollector
|
||||
from typing import List, Set
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class CacheArtifact(TypedDict):
|
||||
array_index: int
|
||||
name: str
|
||||
size: int
|
||||
|
||||
|
||||
class CacheMetrics(TypedDict):
|
||||
cache_artifacts: List[CacheArtifact]
|
||||
push_time: int
|
||||
pull_time: int
|
||||
cache_size: int
|
||||
|
||||
|
||||
class CacheDirValidateWithMetrics(CacheDirValidate):
|
||||
DATA: CacheMetrics = CacheMetrics(push_time=0, pull_time=0, cache_artifacts=[], cache_size=0)
|
||||
SET: Set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def keep_stats(cls, target):
|
||||
if target in cls.SET:
|
||||
return
|
||||
|
||||
size = os.path.getsize(target)
|
||||
cls.DATA["cache_artifacts"].append(
|
||||
CacheArtifact(array_index=len(cls.DATA["cache_artifacts"]), name=target, size=size))
|
||||
cls.DATA["cache_size"] += size
|
||||
cls.SET.add(target)
|
||||
|
||||
@classmethod
|
||||
def copy_from_cache(cls, env, src, dst):
|
||||
start = time.time_ns()
|
||||
super().copy_from_cache(env, src, dst)
|
||||
pull_time = time.time_ns() - start
|
||||
cls.DATA["pull_time"] += pull_time
|
||||
cls.keep_stats(dst)
|
||||
|
||||
@classmethod
|
||||
def copy_to_cache(cls, env, src, dst):
|
||||
start = time.time_ns()
|
||||
super().copy_to_cache(env, src, dst)
|
||||
push_time = time.time_ns() - start
|
||||
cls.DATA["push_time"] += push_time
|
||||
cls.keep_stats(src)
|
||||
|
||||
|
||||
class CacheDirCollector(BuildMetricsCollector):
|
||||
def get_name(self):
|
||||
return "CacheDirCollector"
|
||||
|
||||
def finalize(self):
|
||||
return "cache_metrics", CacheDirValidateWithMetrics.DATA
|
||||
Loading…
Reference in New Issue