diff --git a/.agents/skills/_patterns.md b/.agents/skills/_patterns.md new file mode 100644 index 00000000..429e1395 --- /dev/null +++ b/.agents/skills/_patterns.md @@ -0,0 +1,918 @@ +# Claude Skills Common Patterns + +This document contains reusable patterns referenced by AI Software Architect skills. These patterns promote consistency and reduce duplication across skills. + +**Note**: This is a reference document, not a skill itself. It does not have YAML frontmatter. + +## Table of Contents + +1. [Tool Permission Pattern](#tool-permission-pattern) +2. [Progressive Disclosure Pattern](#progressive-disclosure-pattern) +3. [Input Validation & Sanitization](#input-validation--sanitization) +4. [Error Handling](#error-handling) +5. [File Loading](#file-loading) +6. [Reporting Format](#reporting-format) +7. [Skill Workflow Template](#skill-workflow-template) + +--- + +## Tool Permission Pattern + +**Status**: ✅ Implemented across all skills (2025-12-04) +**Reference**: [ADR-007](../../.architecture/decisions/adrs/ADR-007-tool-permission-restrictions-for-skills.md) + +All skills MUST declare tool permissions via `allowed-tools` in YAML frontmatter. This implements principle of least privilege and limits blast radius of skill malfunctions. + +### Available Tools + +- **Read**: Read files from filesystem +- **Write**: Create new files +- **Edit**: Modify existing files +- **Glob**: Pattern-based file searching +- **Grep**: Content searching within files +- **Bash**: Execute bash commands (can be scoped) + +### Permission Guidelines + +**Principle of Least Privilege**: Only grant tools actually required for skill operation. + +**Bash Scoping**: Use wildcards to restrict bash commands: +- `Bash(git:*)` - Only git commands +- `Bash(ls:*,grep:*)` - Only ls and grep +- `Bash(npm:*,node:*)` - Only npm and node +- `Bash` - Full bash access (use sparingly) + +### Permission Levels by Skill Type + +**Read-Only Skills** (listing, status checks): +```yaml +allowed-tools: Read +``` + +**Search & Analysis Skills** (scanning, reporting): +```yaml +allowed-tools: Read,Glob,Grep +``` + +**Document Creation Skills** (ADRs, reviews): +```yaml +allowed-tools: Read,Write,Bash(ls:*,grep:*) +``` + +**Document Modification Skills** (updates, edits): +```yaml +allowed-tools: Read,Write,Edit,Glob,Grep +``` + +**Review & Analysis Skills** (comprehensive reviews): +```yaml +allowed-tools: Read,Write,Glob,Grep,Bash(git:*) +``` + +**Configuration Skills** (mode toggles): +```yaml +allowed-tools: Read,Edit +``` + +**Setup & Installation Skills** (framework setup): +```yaml +allowed-tools: Read,Write,Edit,Glob,Grep,Bash +``` + +### Current Skill Permissions + +| Skill | Tools | Rationale | +|-------|-------|-----------| +| `list-members` | `Read` | Only reads members.yml | +| `architecture-status` | `Read,Glob,Grep` | Scans files and searches | +| `create-adr` | `Read,Write,Bash(ls:*,grep:*)` | Creates ADRs, scans for numbering | +| `specialist-review` | `Read,Write,Glob,Grep` | Reviews code, writes reports | +| `architecture-review` | `Read,Write,Glob,Grep,Bash(git:*)` | Full reviews + git status | +| `pragmatic-guard` | `Read,Edit` | Reads/modifies config | +| `setup-architect` | `Read,Write,Edit,Glob,Grep,Bash` | Full installation access | + +### Adding Permissions to New Skills + +When creating a new skill, determine required tools: + +1. **List required operations**: + - Need to read files? → `Read` + - Need to create files? → `Write` + - Need to modify files? → `Edit` + - Need to search by pattern? → `Glob` + - Need to search content? → `Grep` + - Need bash commands? → `Bash(scoped:*)` or `Bash` + +2. **Apply minimum necessary set**: + - Start with minimal permissions + - Add only when operation actually requires it + - Scope Bash to specific command families + +3. **Add to frontmatter**: + ```yaml + --- + name: skill-name + description: ... + allowed-tools: Read,Write,Grep + --- + ``` + +4. **Test thoroughly**: + - Verify skill can perform all operations + - Confirm no permission errors + - Test edge cases + +### Security Considerations + +**Path Traversal**: Tools like Read, Write, Edit are bounded by Claude Code's security model, but skills should still validate user inputs. + +**Command Injection**: When using Bash, always sanitize user inputs using filename sanitization patterns. + +**Wildcard Safety**: Bash scoping reduces but doesn't eliminate risks. Still apply input validation. + +**Audit Regularly**: Review tool permissions when modifying skills to ensure they still follow least privilege. + +### Troubleshooting Permission Errors + +**Error: "Tool X not allowed"** +- Check skill's `allowed-tools` frontmatter +- Add required tool if operation is legitimate +- Consider if operation can be done with allowed tools + +**Error: "Bash command not allowed"** +- If using `Bash(scoped:*)`, ensure command matches scope +- Consider broadening scope: `Bash(git:*,npm:*)` +- As last resort, use full `Bash` access (document why) + +**Permission Too Broad?** +- Review actual tool usage in skill +- Remove unused tools from `allowed-tools` +- Narrow Bash scoping if possible + +--- + +## Progressive Disclosure Pattern + +**Status**: ✅ Implemented (Phase 2 Complete - 2025-12-04) +**Reference**: [ADR-008](../../.architecture/decisions/adrs/ADR-008-progressive-disclosure-pattern-for-large-skills.md) + +Organize complex skills into modular structure: high-level workflow + detailed references + templates. This pattern improves token efficiency, enables content expansion, and dramatically improves maintainability. + +### When to Use Progressive Disclosure + +**Apply pattern when:** +- Skill has distinct workflow vs detailed guidance sections +- Skill >2K words or contains extensive procedural detail +- Skill includes templates that could be extracted +- Future expansion anticipated (tech stacks, examples, checklists) +- Clear separation would improve navigability + +**Keep as flat file when:** +- Skill <2K words with homogeneous content +- Simple reporting or trivial operation +- No clear workflow vs detail separation +- Overhead exceeds benefits + +### Pattern Structure + +``` +skill-name/ +├── SKILL.md # High-level workflow (always loaded) +├── references/ # Detailed docs (loaded on-demand) +│ ├── detailed-process.md # Step-by-step procedures +│ └── integration-guide.md # Configuration and customization +└── assets/ # Templates and static files + └── template.md # Ready-to-use templates +``` + +### SKILL.md (Base Workflow) + +**Purpose**: High-level workflow that's always injected into context + +**Content**: +- YAML frontmatter (name, description, allowed-tools) +- Overview of skill purpose +- High-level workflow steps (numbered) +- Clear references to detailed docs +- Related skills and workflow examples + +**Size Target**: Keep concise - aim for 500-1000 words + +**Example Structure**: +```markdown +--- +name: skill-name +description: Clear description with trigger phrases +allowed-tools: Read,Write,Glob,Grep +--- + +# Skill Title + +Brief description of what this skill does. + +## Overview + +High-level summary (3-5 bullets) + +**Detailed guidance**: [references/detailed-process.md](references/detailed-process.md) +**Template**: [assets/template.md](assets/template.md) + +## High-Level Workflow + +### 1. [Step Name] +Brief description +- Key action +- Key action + +**Detailed procedures**: See [references/detailed-process.md § Step 1] + +### 2. [Step Name] +Brief description + +[Continue with high-level steps] + +## Related Skills +[References to before/after skills] +``` + +### references/ (Detailed Documentation) + +**Purpose**: Comprehensive guidance loaded only when needed + +**Content Types**: +- **Process details**: Step-by-step procedures with verification +- **Integration guides**: Configuration options and customization +- **Specialist guidance**: Expert perspectives and checklists +- **Best practices**: Industry standards and patterns +- **Examples**: Code samples, scenarios, use cases + +**Organization**: +- 1-3 focused reference files per skill +- Each file covers a distinct aspect +- Clear section headers for easy navigation +- Cross-reference to templates when relevant + +**Example references/detailed-process.md**: +```markdown +# Detailed Process: [Skill Name] + +## Prerequisites + +[Requirements and verification steps] + +## Step 1: [Name] + +**Purpose**: [Why this step] + +**Procedure**: +1. [Action] +2. [Action with bash example if needed] +3. [Verification step] + +**Common Issues**: +- [Issue]: [Solution] + +**Example**: +[Code or command example] + +[Continue with remaining steps in detail] +``` + +### assets/ (Templates) + +**Purpose**: Ready-to-use templates and static files + +**Content Types**: +- Document templates (ADRs, reviews, reports) +- Configuration templates (YAML, JSON) +- Code snippets or boilerplate + +**Format**: +- Complete, fillable templates +- Include placeholder text in [brackets] +- Add comments explaining each section +- Provide examples where helpful + +**Example assets/template.md**: +```markdown +# [Template Title] + +**[Field]**: [Value or placeholder] +**[Field]**: [Value] + +## [Section Name] + +[Guidance for filling this section] + +[Example or placeholder content] + +[Continue with complete template structure] +``` + +### Phase 2 Results (Proven Benefits) + +**Skills Refactored**: 3 (architecture-review, setup-architect, specialist-review) + +**Token Efficiency**: +- Variable by starting optimization: 0-34% base reduction +- Average: 13% base reduction across 3 skills +- Annual savings: ~18,380 tokens/year (estimated) + +**Content Expansion**: +- Consistent 300%+ increase per skill +- architecture-review: +375% (791 → 3,757 words) +- setup-architect: +494% (813 → 4,837 words) +- specialist-review: +332% (826 → 3,571 words) +- Average: +400% content expansion + +**Maintainability**: +- 9 new reference and asset files created +- Clear separation: workflow vs procedures vs templates +- Isolated updates without affecting core workflow +- Dramatically improved navigation + +**Key Insight**: Pattern delivers value through multiple dimensions beyond token savings. Even skills with no base reduction gain significant value through content expansion and improved maintainability. + +### Implementation Guidelines + +**1. Analyze Current Skill**: +- Identify workflow steps (high-level) +- Identify detailed procedures (low-level) +- Identify templates embedded inline +- Determine natural separation points + +**2. Create Directory Structure**: +```bash +mkdir -p .claude/skills/skill-name/references +mkdir -p .claude/skills/skill-name/assets +``` + +**3. Extract Content**: +- **To SKILL.md**: Keep workflow, overview, high-level steps +- **To references/**: Move detailed procedures, comprehensive guides +- **To assets/**: Extract templates, configurations + +**4. Add References**: +- Link from SKILL.md to references with relative paths +- Use descriptive section anchors: `references/file.md § Section` +- Keep references concise and navigable + +**5. Test Loading**: +- Verify all reference links work +- Test skill execution with minimal loading +- Test skill execution with full reference loading +- Confirm templates are accessible + +**6. Measure Results**: +- Count words: before vs after (base and total) +- Estimate token savings +- Assess maintainability improvement +- Document in comparisons/ + +### Refactoring Checklist + +- [ ] Skill has clear workflow vs detail separation +- [ ] Created /references/ and /assets/ directories +- [ ] Extracted detailed procedures to references/ +- [ ] Extracted templates to assets/ +- [ ] Streamlined SKILL.md to high-level workflow +- [ ] Added clear references from SKILL.md to detailed docs +- [ ] Updated YAML frontmatter (name, description, allowed-tools) +- [ ] Tested skill execution +- [ ] Verified reference links work +- [ ] Measured before/after metrics (words, tokens) +- [ ] Documented results in .architecture/comparisons/ +- [ ] Updated ARCHITECTURE.md with new structure + +### Best Practices + +**Do**: +- Keep SKILL.md focused on workflow +- Make references/ comprehensive and detailed +- Extract templates to assets/ for reusability +- Use clear, descriptive file names +- Cross-reference between files appropriately +- Test all links after refactoring + +**Don't**: +- Duplicate content between SKILL.md and references/ +- Over-fragment (too many small reference files) +- Break mid-workflow (keep logical steps together) +- Forget to update allowed-tools in frontmatter +- Skip measuring and documenting results + +### Migration Path + +**Existing flat file skill** → **Progressive disclosure**: + +1. Read current SKILL.md and analyze content +2. Create directory structure (references/, assets/) +3. Extract detailed content to references/ +4. Extract templates to assets/ +5. Rewrite SKILL.md as high-level workflow with references +6. Test and verify +7. Measure and document results + +**Estimated effort**: 2-3 hours per skill + +### References + +- [Phase 2 PoC Results](../../.architecture/comparisons/progressive-disclosure-poc-results.md) +- [Phase 2A Results](../../.architecture/comparisons/phase-2a-setup-architect-results.md) +- [Phase 2B Results](../../.architecture/comparisons/phase-2b-specialist-review-results.md) +- [Phase 2 Complete Summary](../../.architecture/comparisons/phase-2-complete-summary.md) +- [ADR-008](../../.architecture/decisions/adrs/ADR-008-progressive-disclosure-pattern-for-large-skills.md) + +--- + +## Input Validation & Sanitization + +### Filename Sanitization Pattern + +Use when converting user input to filenames: + +``` +**Validate and Sanitize Input**: +- Remove path traversal: `..`, `/`, `\` +- Remove dangerous characters: null bytes, control characters +- Convert to lowercase kebab-case: + - Spaces → hyphens + - Remove special characters except hyphens and alphanumerics +- Limit length: max 80-100 characters +- Validate result matches: [a-z0-9-] pattern + +**Examples**: +✅ Valid: "User Authentication" → `user-authentication` +✅ Valid: "React Frontend" → `react-frontend` +❌ Invalid blocked: "../../../etc/passwd" → rejected +❌ Invalid blocked: "test\x00file" → rejected +``` + +### Version Number Validation Pattern + +Use for semantic version numbers: + +``` +**Version Validation**: +- Format: X.Y.Z (e.g., 1.2.3) +- Allow only: digits (0-9) and dots (.) +- Validate: 1-3 numeric segments separated by dots +- Convert dots to hyphens for filenames: 1.2.3 → 1-2-3 + +**Examples**: +✅ Valid: "2.1.0" → `2-1-0` +✅ Valid: "1.0" → `1-0` +❌ Invalid: "v2.1.0" → strip 'v' prefix +❌ Invalid: "2.1.0-beta" → reject or sanitize +``` + +### Specialist Role Validation Pattern + +Use for specialist role names: + +``` +**Role Validation**: +- Allow: letters, numbers, spaces, hyphens +- Convert to title case for display +- Convert to kebab-case for filenames +- Common roles: Security Specialist, Performance Expert, Domain Expert + +**Examples**: +✅ Valid: "Security Specialist" → display as-is, file: `security-specialist` +✅ Valid: "Ruby Expert" → display as-is, file: `ruby-expert` +❌ Invalid: "Security/Admin" → sanitize to `security-admin` +``` + +--- + +## Error Handling + +### Framework Not Set Up Pattern + +Use when .architecture/ doesn't exist: + +```markdown +The AI Software Architect framework is not set up yet. + +To get started: "Setup ai-software-architect" + +Once set up, you'll have: +- Architectural Decision Records (ADRs) +- Architecture reviews with specialized perspectives +- Team of architecture specialists +- Documentation tracking and status monitoring +``` + +### File Not Found Pattern + +Use when required files are missing: + +```markdown +Could not find [file/directory name]. + +This usually means: +1. Framework may not be set up: "Setup ai-software-architect" +2. File was moved or deleted +3. Wrong directory + +Expected location: [path] +``` + +### Permission Error Pattern + +Use for file system permission issues: + +```markdown +Permission denied accessing [file/path]. + +Please check: +1. File permissions: chmod +r [file] +2. Directory permissions: chmod +rx [directory] +3. You have access to this project directory +``` + +### Malformed YAML Pattern + +Use when YAML parsing fails: + +```markdown +Error reading [file]: YAML syntax error + +Common issues: +- Incorrect indentation (use spaces, not tabs) +- Missing quotes around special characters +- Unclosed strings or brackets + +Please check file syntax or restore from template: +[path to template] +``` + +--- + +## File Loading + +### Load Configuration Pattern + +Use for loading config.yml: + +``` +1. Check if `.architecture/config.yml` exists +2. If missing: Use default configuration (pragmatic_mode: disabled) +3. If exists: Parse YAML +4. Extract relevant settings: + - pragmatic_mode.enabled (boolean) + - pragmatic_mode.intensity (strict|balanced|lenient) + - Other mode-specific settings +5. Handle errors gracefully (malformed YAML → use defaults) +``` + +### Load Members Pattern + +Use for loading members.yml: + +``` +1. Check if `.architecture/members.yml` exists +2. If missing: Offer framework setup +3. If exists: Parse YAML +4. Extract member information: + - id, name, title (required) + - specialties, disciplines, skillsets, domains (arrays) + - perspective (string) +5. Validate structure (warn about missing fields) +6. Return array of member objects +``` + +### Load ADR List Pattern + +Use for scanning ADR directory: + +``` +1. Check `.architecture/decisions/adrs/` exists +2. List files matching: ADR-[0-9]+-*.md +3. Extract ADR numbers and titles from filenames +4. Sort by ADR number (numeric sort) +5. Optionally read file headers for: + - Status (Proposed, Accepted, Deprecated, Superseded) + - Date + - Summary +6. Return sorted list with metadata +``` + +--- + +## Reporting Format + +### Success Report Pattern + +Use after successfully completing a skill task: + +```markdown +[Skill Action] Complete: [Target] + +Location: [file path] +[Key metric]: [value] + +Key Points: +- [Point 1] +- [Point 2] +- [Point 3] + +Next Steps: +- [Action 1] +- [Action 2] +``` + +**Example**: +```markdown +ADR Created: Use PostgreSQL Database + +Location: .architecture/decisions/adrs/ADR-005-use-postgresql.md +Status: Accepted + +Key Points: +- Decision: PostgreSQL over MySQL for JSONB support +- Main benefit: Better performance for semi-structured data +- Trade-off: Team needs PostgreSQL expertise + +Next Steps: +- Review with Performance Specialist +- Update deployment documentation +- Plan migration timeline +``` + +### Status Report Pattern + +Use for providing status/health information: + +```markdown +# [Status Type] Report + +**Report Date**: [Date] +**Health Status**: Excellent | Good | Needs Attention | Inactive + +## Summary + +**Key Metrics**: +- [Metric 1]: [value] +- [Metric 2]: [value] +- [Metric 3]: [value] + +## Detailed Findings + +[Sections with specific information] + +## Recommendations + +[Actionable next steps based on current state] +``` + +### Review Report Pattern + +Use for architecture and specialist reviews: + +```markdown +# [Review Type]: [Target] + +**Reviewer**: [Name/Role] +**Date**: [Date] +**Assessment**: Excellent | Good | Adequate | Needs Improvement | Critical Issues + +## Executive Summary +[2-3 sentences] + +**Key Findings**: +- [Finding 1] +- [Finding 2] + +## [Detailed Analysis Sections] + +## Recommendations + +### Immediate (0-2 weeks) +1. **[Action]**: [Details] + +### Short-term (2-8 weeks) +1. **[Action]**: [Details] + +### Long-term (2-6 months) +1. **[Action]**: [Details] +``` + +--- + +## Skill Workflow Template + +### Standard Skill Structure + +All skills should follow this structure: + +```markdown +--- +name: skill-name +description: Clear description with trigger phrases. Use when... Do NOT use for... +allowed-tools: [Read, Write, Edit, Glob, Grep, Bash] # Optional: restrict for security +--- + +# Skill Title + +One-line description of what this skill does. + +## Process + +### 1. [First Step] +- Action item +- Action item + +### 2. [Second Step] +- Action item +- Action item + +[Continue with numbered steps] + +### N. Report to User +[Use appropriate reporting pattern] + +## [Optional Sections] + +### When to Use +- Scenario 1 +- Scenario 2 + +### When NOT to Use +- Scenario 1 +- Scenario 2 + +## Related Skills + +**Before This Skill**: +- "[Related skill]" - [Why] + +**After This Skill**: +- "[Related skill]" - [Why] + +**Workflow Examples**: +1. [Skill chain example 1] +2. [Skill chain example 2] + +## Error Handling +- [Error type]: [How to handle] +- [Error type]: [How to handle] + +## Notes +- Implementation note +- Best practice +- Important consideration +``` + +--- + +## Destructive Operations Safety Pattern + +Use for operations that delete or modify files irreversibly: + +``` +**CRITICAL SAFEGUARDS**: +1. Verify current directory context + - Check for project markers (package.json, .git, README.md, etc.) + - Confirm we're in expected location + +2. Verify target exists and is correct + - Check file/directory exists: `[ -e /path/to/target ]` + - Verify it's what we expect (check contents or structure) + +3. Verify target is safe to modify/delete + - For .git removal: verify it's template repo, not project repo + - Check .git/config contains expected template URL + - Ensure no uncommitted work or important history + +4. Use absolute paths + - Get absolute path: `$(pwd)/relative/path` + - Never use relative paths with rm -rf + +5. Never use wildcards + - ❌ Bad: `rm -rf .architecture/.git*` + - ✅ Good: `rm -rf $(pwd)/.architecture/.git` + +6. Stop and ask if verification fails + - **STOP AND ASK USER** if any check fails + - Explain what failed and why it's unsafe + - Let user confirm or abort + +**Example Safe Deletion**: +```bash +# 1. Verify we're in project root +if [ ! -f "package.json" ] && [ ! -f ".git/config" ]; then + echo "ERROR: Not in project root" + exit 1 +fi + +# 2. Verify target exists +if [ ! -d ".architecture/.git" ]; then + echo "Nothing to remove" + exit 0 +fi + +# 3. Verify it's the template repo +if ! grep -q "ai-software-architect" .architecture/.git/config 2>/dev/null; then + echo "ERROR: .architecture/.git doesn't appear to be template repo" + echo "STOPPING - User confirmation required" + exit 1 +fi + +# 4. Safe removal with absolute path +rm -rf "$(pwd)/.architecture/.git" +``` +``` + +--- + +## Directory Structure Validation Pattern + +Use when skills need specific directory structures: + +``` +**Directory Structure Check**: +1. Check `.architecture/` exists + - If missing: Suggest "Setup ai-software-architect" + +2. Check required subdirectories: + - `.architecture/decisions/adrs/` + - `.architecture/reviews/` + - `.architecture/templates/` + - `.architecture/recalibration/` + - `.architecture/comparisons/` + +3. Create missing subdirectories if skill will use them + - Use: `mkdir -p .architecture/[subdirectory]` + +4. Verify key files exist: + - `.architecture/members.yml` + - `.architecture/principles.md` + - `.architecture/config.yml` (optional, use defaults if missing) + +5. Report issues clearly: + - Missing directories: Create them + - Missing required files: Suggest setup or provide template + - Permission issues: Report and suggest fixes +``` + +--- + +## Usage Notes + +### How to Reference Patterns in Skills + +In skill files, reference patterns like this: + +```markdown +### 3. Validate Input +See [Input Validation & Sanitization](#input-validation--sanitization) in _patterns.md. + +Apply filename sanitization pattern to user-provided title. +``` + +### When to Add New Patterns + +Add new patterns when: +1. Same logic appears in 3+ skills +2. Pattern solves a common problem +3. Pattern improves security or reliability +4. Pattern promotes consistency + +### When NOT to Extract Patterns + +Don't extract when: +1. Logic is skill-specific +2. Pattern would be more complex than inline code +3. Pattern only used in 1-2 skills +4. Extraction reduces clarity + +--- + +## Version History + +**v1.2** (2025-12-04) +- Added progressive disclosure pattern (ADR-008) +- Documented modular skill structure (SKILL.md + /references/ + /assets/) +- Included Phase 2 proven results and metrics +- Added implementation guidelines and refactoring checklist +- Provided best practices for applying pattern +- Added migration path for existing skills + +**v1.1** (2025-12-04) +- Added tool permission pattern (ADR-007) +- Documented `allowed-tools` frontmatter requirement +- Added permission guidelines by skill type +- Added Bash scoping examples +- Added security considerations for tool permissions +- Added troubleshooting guide for permission errors + +**v1.0** (2025-11-12) +- Initial patterns document +- Input validation patterns +- Error handling patterns +- File loading patterns +- Reporting format patterns +- Skill workflow template +- Destructive operations safety pattern +- Directory structure validation pattern diff --git a/.agents/skills/architecture-review/SKILL.md b/.agents/skills/architecture-review/SKILL.md new file mode 100644 index 00000000..0a707486 --- /dev/null +++ b/.agents/skills/architecture-review/SKILL.md @@ -0,0 +1,132 @@ +--- +name: architecture-review +description: Conducts a comprehensive multi-perspective architecture review using ALL architecture team members. Use when the user requests "Start architecture review", "Full architecture review", "Review architecture for version X.Y.Z", "Conduct comprehensive review", or when they want assessment from multiple perspectives. Do NOT use for single-specialist reviews (use specialist-review instead) or for status checks (use architecture-status instead). +allowed-tools: Read,Write,Glob,Grep,Bash(git:*) +--- + +# Architecture Review + +Conducts comprehensive multi-perspective architecture reviews with all team members. + +## Process Overview + +1. **Determine Scope** - Identify what to review (version, feature, or component) +2. **Load Team** - Read members from `.architecture/members.yml` and check pragmatic mode +3. **Analyze System** - Examine architecture using Read, Glob, Grep, and git tools +4. **Individual Reviews** - Each member reviews from their specialized perspective +5. **Collaborative Discussion** - Synthesize findings and establish priorities +6. **Create Document** - Generate comprehensive review using template +7. **Report Results** - Summarize findings and next steps for user + +**Detailed guidance**: [references/review-process.md](references/review-process.md) + +## Workflow Steps + +### 1. Determine Scope + +Identify review target and create filename: +- **Version**: "version X.Y.Z" → `X-Y-Z.md` +- **Feature**: "feature name" → `feature-kebab-case.md` +- **Component**: "component name" → `component-kebab-case.md` + +Apply input validation (see `_patterns.md` § Filename Sanitization). + +### 2. Load Configuration and Team + +```bash +cat .architecture/config.yml # Check pragmatic_mode.enabled +cat .architecture/members.yml # Load all members +``` + +Include Pragmatic Enforcer if pragmatic mode enabled for reviews. + +### 3. Analyze the Target + +Use available tools to examine the system: +- `Read` - Code, configs, documentation +- `Glob` - Find files by pattern +- `Grep` - Search for specific patterns +- `Bash(git:*)` - Git history and status + +Focus based on review type: +- **Version**: Overall architecture, components, patterns, technical debt +- **Feature**: Implementation, integration, security, performance +- **Component**: Structure, dependencies, boundaries, interfaces + +### 4. Conduct Individual Member Reviews + +For each member in `members.yml`, write a review including: +- Perspective statement +- Key observations (3-5) +- Strengths (3-5) +- Concerns with impact and recommendations (3-7) +- Prioritized recommendations with effort estimates (3-7) + +**Format details**: [references/review-process.md § Individual Member Review Format](references/review-process.md#individual-member-review-format) + +**Pragmatic integration**: If enabled, add pragmatic analysis after each member. See [references/pragmatic-integration.md](references/pragmatic-integration.md) + +### 5. Facilitate Collaborative Discussion + +Synthesize findings: +- Identify common concerns +- Discuss disagreements +- Establish consensus +- Prioritize: Critical (0-2 weeks) | Important (2-8 weeks) | Nice-to-Have (2-6 months) + +**Discussion format**: [references/review-process.md § Collaborative Discussion](references/review-process.md#collaborative-discussion-process) + +### 6. Create Review Document + +Load template and fill in all sections: +```bash +cat .claude/skills/architecture-review/assets/review-template.md +``` + +Include: +- Executive summary and overall assessment +- Individual member reviews +- Collaborative discussion +- Consolidated findings (strengths, improvements, debt, risks) +- Recommendations (immediate, short-term, long-term) +- Success metrics and follow-up plan + +Save to `.architecture/reviews/[filename].md` + +**Template**: [assets/review-template.md](assets/review-template.md) + +### 7. Report to User + +``` +Architecture Review Complete: [Target] + +Location: .architecture/reviews/[filename].md +Overall Assessment: [Strong | Adequate | Needs Improvement] + +Top 3 Priorities: +1. [Priority 1] +2. [Priority 2] +3. [Priority 3] + +Immediate Actions: +- [Action 1] +- [Action 2] + +Next Steps: +- Review with team +- "Start architecture recalibration for [target]" +- Create ADRs for key decisions +``` + +## Related Skills + +**Before**: `architecture-status`, `list-members` +**During**: `specialist-review`, `create-adr` +**After**: `architecture-recalibration`, `create-adr` + +## Documentation + +- **Process guide**: [references/review-process.md](references/review-process.md) +- **Pragmatic mode**: [references/pragmatic-integration.md](references/pragmatic-integration.md) +- **Template**: [assets/review-template.md](assets/review-template.md) +- **Patterns**: [../_patterns.md](../_patterns.md) diff --git a/.agents/skills/architecture-review/assets/review-template.md b/.agents/skills/architecture-review/assets/review-template.md new file mode 100644 index 00000000..c97e2bf0 --- /dev/null +++ b/.agents/skills/architecture-review/assets/review-template.md @@ -0,0 +1,294 @@ +# Architecture Review: [Target] + +**Date**: [YYYY-MM-DD] +**Review Type**: Version | Feature | Component +**Reviewers**: [List all participating members] + +## Executive Summary + +[Write 2-3 paragraphs summarizing the overall state of the architecture, key findings, and critical actions needed. This should be readable by non-technical stakeholders.] + +**Overall Assessment**: Strong | Adequate | Needs Improvement + +**Key Findings**: +- [Finding 1 - Most significant discovery] +- [Finding 2 - Second most significant] +- [Finding 3 - Third most significant] + +**Critical Actions**: +- [Action 1 - Most urgent action required] +- [Action 2 - Second most urgent] + +--- + +## System Overview + +[Provide context about what was reviewed. Include: +- Version number or feature name +- Scope of review (whole system, specific component, feature) +- Key technologies and frameworks +- Architecture style (monolith, microservices, etc.) +- Team size and structure +- Any relevant constraints or context] + +--- + +## Individual Member Reviews + +[Insert each member's review using the format from references/review-process.md] + +### [Member Name] - [Title] + +**Perspective**: [Their unique viewpoint] + +#### Key Observations +- [Observation 1] +- [Observation 2] + +#### Strengths +1. **[Strength]**: [Description] + +#### Concerns +1. **[Concern]** (Impact: High/Medium/Low) + - **Issue**: [What's wrong] + - **Why it matters**: [Impact] + - **Recommendation**: [What to do] + +#### Recommendations +1. **[Recommendation]** (Priority: High/Medium/Low, Effort: Small/Medium/Large) + - **What**: [Action] + - **Why**: [Benefit] + - **How**: [Approach] + +[If pragmatic mode enabled, add Pragmatic Enforcer Analysis section here - see references/pragmatic-integration.md] + +[Repeat for each member] + +--- + +## Collaborative Discussion + +[Synthesize findings from all members. Show how different perspectives interact and what consensus emerges.] + +**Opening Context**: + +**[Systems Architect]**: "[Opening statement]" + +**[Domain Expert]**: "[Response or complementary view]" + +[Continue natural discussion flow between members] + +### Common Ground + +The team agrees on: +1. [Consensus point 1] +2. [Consensus point 2] +3. [Consensus point 3] + +### Areas of Debate + +**Topic: [Topic Title]** +- **[Member 1]**: [Their position and reasoning] +- **[Member 2]**: [Their position and reasoning] +- **Resolution**: [How team reconciles different views] + +### Priorities Established + +**Critical (Address Immediately)**: +1. [Critical priority with brief justification] +2. [Critical priority] + +**Important (Address Soon)**: +1. [Important priority] +2. [Important priority] + +**Nice-to-Have (Consider Later)**: +1. [Nice-to-have improvement] +2. [Nice-to-have improvement] + +--- + +## Consolidated Findings + +### Strengths + +1. **[Strength Title]**: [Description of what's working well, why it's valuable, and how to sustain it] +2. **[Strength Title]**: [Description] +3. **[Strength Title]**: [Description] + +[Aim for 4-7 key strengths] + +### Areas for Improvement + +1. **[Area Title]**: + - **Current state**: [What exists now] + - **Desired state**: [What it should be] + - **Gap**: [What's missing] + - **Priority**: High | Medium | Low + - **Impact**: [Why this matters] + +2. **[Area Title]**: [Details] + +[Aim for 5-10 areas depending on review scope] + +### Technical Debt + +**High Priority**: +- **[Debt Item]**: + - **Impact**: [How it affects development/operations] + - **Resolution**: [What needs to be done] + - **Effort**: Small | Medium | Large + - **Recommended Timeline**: [When to address] + +**Medium Priority**: +- **[Debt Item]**: [Details] + +**Low Priority**: +- **[Debt Item]**: [Details] + +### Risks + +**Technical Risks**: +- **[Risk Title]** (Likelihood: High/Medium/Low, Impact: High/Medium/Low) + - **Description**: [What could go wrong] + - **Mitigation**: [How to reduce or eliminate risk] + - **Owner**: [Who should monitor/address] + +**Business Risks**: +- **[Risk Title]** (Likelihood: High/Medium/Low, Impact: High/Medium/Low) + - **Description**: [Business impact] + - **Mitigation**: [How to address] + +**Operational Risks**: +- **[Risk Title]** (Likelihood: High/Medium/Low, Impact: High/Medium/Low) + - **Description**: [Operations/reliability concern] + - **Mitigation**: [How to address] + +--- + +## Recommendations + +### Immediate (0-2 weeks) + +1. **[Action Title]** + - **Why**: [Problem being solved or value being created] + - **How**: [High-level implementation approach] + - **Owner**: [Team or person responsible] + - **Success Criteria**: [How to know it's done successfully] + - **Estimated Effort**: [Time or story points] + +2. **[Action Title]**: [Details] + +### Short-term (2-8 weeks) + +1. **[Action Title]** + - **Why**: [Justification] + - **How**: [Approach] + - **Owner**: [Responsible party] + - **Success Criteria**: [Completion criteria] + - **Estimated Effort**: [Effort estimate] + +2. **[Action Title]**: [Details] + +### Long-term (2-6 months) + +1. **[Action Title]** + - **Why**: [Strategic value or risk mitigation] + - **How**: [High-level roadmap] + - **Owner**: [Responsible party] + - **Success Criteria**: [Long-term goals] + - **Estimated Effort**: [Effort estimate] + +2. **[Action Title]**: [Details] + +--- + +## Success Metrics + +Define measurable criteria to track improvement: + +1. **[Metric Name]**: + - **Current**: [Current value] + - **Target**: [Target value] + - **Timeline**: [When to achieve] + - **Measurement**: [How to measure] + +2. **[Metric Name]**: Current → Target (Timeline) + +3. **[Metric Name]**: Current → Target (Timeline) + +[Examples: +- Test Coverage: 45% → 75% (3 months) +- Build Time: 15 min → 5 min (6 weeks) +- P95 Response Time: 500ms → 200ms (2 months) +- Code Review Time: 2 days → 4 hours (1 month) +] + +--- + +## Follow-up + +**Next Review**: [Specific date or milestone] + +**Tracking**: [How recommendations will be tracked] +- Create GitHub issues for immediate actions +- Add to sprint backlog +- Use architecture recalibration process (see below) + +**Recalibration**: +After implementing recommendations, conduct architecture recalibration to assess progress: +``` +"Start architecture recalibration for [target]" +``` + +**Accountability**: +- [Who is responsible for tracking implementation] +- [How often to check progress - weekly, bi-weekly, etc.] +- [Where to document status updates] + +--- + +## Related Documentation + +**Architectural Decision Records**: +- [ADR-XXX: Title](../decisions/adrs/ADR-XXX-title.md) - [How it relates to this review] +- [ADR-YYY: Title](../decisions/adrs/ADR-YYY-title.md) - [Relationship] + +**Previous Reviews**: +- [Previous review filename] - [Date] - [How architecture has evolved since then] + +**Referenced Documents**: +- [Document title] - [Link] - [Relevance] + +--- + +## Appendix + +### Review Methodology + +This review was conducted using the AI Software Architect framework with the following team members: + +- **Systems Architect**: Overall system coherence and patterns +- **Domain Expert**: Business domain representation +- **Security Specialist**: Security analysis and threat modeling +- **Performance Specialist**: Performance and scalability +- **Maintainability Expert**: Code quality and technical debt +- [Additional members as applicable] + +Each member reviewed independently, then collaborated to synthesize findings and prioritize recommendations. + +[If pragmatic mode was enabled, note:] +**Pragmatic Mode**: [Strict | Balanced | Lenient] +- Complexity ratio target: [<1.0 | <1.5 | <2.0] +- All recommendations evaluated through YAGNI lens + +### Glossary + +[If needed, define domain-specific terms or acronyms used in the review] + +- **[Term]**: [Definition] +- **[Acronym]**: [Expansion and meaning] + +--- + +**Review Complete** diff --git a/.agents/skills/architecture-review/references/pragmatic-integration.md b/.agents/skills/architecture-review/references/pragmatic-integration.md new file mode 100644 index 00000000..169d152c --- /dev/null +++ b/.agents/skills/architecture-review/references/pragmatic-integration.md @@ -0,0 +1,298 @@ +# Pragmatic Mode Integration for Architecture Reviews + +This document describes how to integrate Pragmatic Enforcer analysis when pragmatic mode is enabled in `.architecture/config.yml`. + +## When to Include Pragmatic Analysis + +**Check Configuration**: Read `.architecture/config.yml`: + +```yaml +pragmatic_mode: + enabled: true + intensity: strict | balanced | lenient + applies_to: + - reviews + - adrs + - implementation +``` + +**If `pragmatic_mode.enabled == true` AND `reviews` is in `applies_to`**: Include Pragmatic Enforcer as a reviewer. + +--- + +## Pragmatic Enforcer Review Format + +The Pragmatic Enforcer reviews each member's recommendations through a YAGNI (You Aren't Gonna Need It) lens. + +### After Each Member's Review + +Add this analysis section immediately following each member's recommendations: + +```markdown +#### Pragmatic Enforcer Analysis + +**Reviewer**: Pragmatic Enforcer +**Mode**: [Strict | Balanced | Lenient] + +[For each recommendation from this member, apply pragmatic analysis] + +##### Recommendation: [Recommendation Title] + +**Necessity Assessment** (0-10): +- **Current need**: [Is this addressing a current, concrete problem?] +- **Future need**: [Likelihood and timeframe for actually needing this] +- **Cost of waiting**: [What breaks if we defer? What's the cost of implementing later?] +- **Evidence of need**: [What concrete evidence justifies this now?] +- **Score**: [X/10] + +**Complexity Assessment** (0-10): +- **Added complexity**: [What complexity does this introduce?] +- **Maintenance burden**: [Ongoing cost to maintain this] +- **Learning curve**: [Impact on team understanding] +- **Dependencies introduced**: [New dependencies or abstractions] +- **Score**: [X/10] + +**Simpler Alternative**: +[Propose a simpler approach that meets current actual requirements, or explain why this is already minimal] + +**Pragmatic Recommendation**: +- ✅ **Implement now**: [Clear current need, appropriate complexity] +- ⚠️ **Simplified version**: [Implement minimal version, defer bells and whistles] +- ⏸️ **Defer**: [Wait for trigger conditions] +- ❌ **Skip**: [Not needed, over-engineering] + +**Justification**: [Clear reasoning for recommendation] + +**If Deferring**: +- **Trigger conditions**: [What would trigger implementing this?] +- **Minimal viable alternative**: [What's the simplest thing that could work now?] + +**Pragmatic Score**: +- Necessity: [X/10] +- Complexity: [X/10] +- Ratio: [Complexity/Necessity = X.X] + +**Target Ratios by Mode**: +- Strict: <1.0 (complexity should be less than necessity) +- Balanced: <1.5 (moderate complexity acceptable for clear need) +- Lenient: <2.0 (higher complexity tolerated for strategic value) + +**Assessment**: [Appropriate engineering | Over-engineering | Under-engineering] +``` + +--- + +## Pragmatic Mode Intensity Levels + +### Strict Mode (Ratio Target: <1.0) + +**Philosophy**: Only implement what's absolutely necessary right now. + +**Approach**: +- Challenge every abstraction +- Defer anything speculative +- Require concrete evidence of current need +- Favor simple, direct solutions +- Question best practices if not clearly applicable + +**Use When**: Tight timeline, small team, MVP, prototyping + +### Balanced Mode (Ratio Target: <1.5) + +**Philosophy**: Implement current needs thoughtfully with reasonable future considerations. + +**Approach**: +- Allow appropriate abstractions +- Defer obvious speculation +- Accept moderate complexity for clear benefits +- Balance pragmatism with code quality +- Apply best practices when they add clear value + +**Use When**: Normal development, sustainable pace, balanced priorities + +### Lenient Mode (Ratio Target: <2.0) + +**Philosophy**: Allow strategic complexity for longer-term benefits. + +**Approach**: +- Permit forward-looking designs +- Accept abstractions for anticipated growth +- Allow higher complexity for strategic value +- Consider longer time horizons +- Apply best practices proactively + +**Use When**: Building platforms, long-term projects, infrastructure + +--- + +## Examples of Pragmatic Analysis + +### Example 1: Caching Layer Recommendation (Balanced Mode) + +**Original Recommendation**: "Add Redis caching layer for all database queries" + +**Pragmatic Analysis**: + +**Necessity Assessment**: 7/10 +- Current need: Page load times are 500ms, target is <200ms +- Future need: Traffic growing 20% monthly +- Cost of waiting: User experience degrading, churn risk +- Evidence: Metrics show 60% of queries are duplicate reads +- Score: 7/10 + +**Complexity Assessment**: 6/10 +- Added complexity: New Redis dependency, cache invalidation logic +- Maintenance: Cache coherency, Redis ops, monitoring +- Learning curve: Team familiar with Redis +- Dependencies: Redis server, connection pooling +- Score: 6/10 + +**Simpler Alternative**: +Start with application-level caching (LRU cache in memory) for hot queries only. Add Redis when in-memory is insufficient. + +**Pragmatic Recommendation**: ⚠️ **Simplified version** + +**Justification**: Current need is clear (7/10) but complexity is moderate (6/10). Start simpler with in-memory caching for most impactful queries. This delivers 80% of benefit with 30% of complexity. Upgrade to Redis when evidence shows memory caching is insufficient. + +**Pragmatic Score**: +- Necessity: 7/10 +- Complexity: 6/10 +- Ratio: 0.86 (<1.5 threshold = acceptable if simplified) + +**Assessment**: Appropriate engineering with simplification + +--- + +### Example 2: Microservices Architecture (Balanced Mode) + +**Original Recommendation**: "Split monolith into 12 microservices" + +**Pragmatic Analysis**: + +**Necessity Assessment**: 3/10 +- Current need: Monolith works, no deployment bottlenecks +- Future need: "Might scale better" - speculative +- Cost of waiting: None - can extract services when bottlenecks emerge +- Evidence: No concrete scaling problems, 2-person team +- Score: 3/10 + +**Complexity Assessment**: 9/10 +- Added complexity: Service boundaries, inter-service communication, distributed tracing, API versioning +- Maintenance: 12 services to deploy, monitor, debug +- Learning curve: Team inexperienced with distributed systems +- Dependencies: Service mesh, API gateway, distributed monitoring +- Score: 9/10 + +**Simpler Alternative**: +Keep monolith. Extract services only when you have concrete evidence of bottlenecks. Start with 1-2 services for proven pain points. + +**Pragmatic Recommendation**: ❌ **Skip** + +**Justification**: Massive complexity (9/10) with minimal current need (3/10). This is premature optimization and over-engineering. Wait for actual scaling problems before adding distributed systems complexity. + +**If Deferring**: +- **Trigger conditions**: Deploy time >30 min, scaling bottlenecks measured, >5 developers working in same code +- **Minimal alternative**: Keep monolith, use modules for internal boundaries + +**Pragmatic Score**: +- Necessity: 3/10 +- Complexity: 9/10 +- Ratio: 3.0 (>> 1.5 threshold = over-engineering) + +**Assessment**: Over-engineering - defer + +--- + +### Example 3: Security Vulnerability Fix (Any Mode) + +**Original Recommendation**: "Fix SQL injection vulnerability in login endpoint" + +**Pragmatic Analysis**: + +**Necessity Assessment**: 10/10 +- Current need: Critical security vulnerability +- Future need: N/A - needed now +- Cost of waiting: Data breach, compliance violation +- Evidence: Security audit found SQL injection +- Score: 10/10 + +**Complexity Assessment**: 2/10 +- Added complexity: Use parameterized queries (standard practice) +- Maintenance: Reduces maintenance risk +- Learning curve: None - team knows parameterized queries +- Dependencies: None +- Score: 2/10 + +**Simpler Alternative**: None - this IS the simple solution + +**Pragmatic Recommendation**: ✅ **Implement now** + +**Justification**: Critical necessity (10/10) with minimal complexity (2/10). No debate - implement immediately. + +**Pragmatic Score**: +- Necessity: 10/10 +- Complexity: 2/10 +- Ratio: 0.2 (<< 1.5 threshold) + +**Assessment**: Essential engineering + +--- + +## Integration in Collaborative Discussion + +During collaborative discussion, the Pragmatic Enforcer should: + +1. **Challenge Complexity**: Question whether proposed solutions are minimal +2. **Require Evidence**: Ask for concrete evidence of need, not speculation +3. **Propose Simplifications**: Suggest simpler alternatives for consideration +4. **Balance Views**: Acknowledge when complexity is justified +5. **Set Priorities**: Help team focus on high-impact, low-complexity wins + +### Example Discussion Snippet + +```markdown +**Security Specialist**: "We should implement zero-trust architecture across all services." + +**Pragmatic Enforcer**: "I see the security value, but that's significant complexity. What's the current threat? Can we start with authentication boundaries on public APIs first, then expand if we see attack patterns?" + +**Security Specialist**: "Fair point. We have had unauthorized access attempts on the admin API but internal services haven't been targeted. Starting with API gateway auth would address 90% of current risk." + +**Systems Architect**: "Agreed - that's a more measured approach. Zero-trust can be part of our roadmap once we have evidence it's needed." +``` + +--- + +## Exemptions from Pragmatic Analysis + +Certain recommendations should NOT be challenged by pragmatic mode: + +1. **Security Vulnerabilities**: Critical security fixes always implement +2. **Compliance Requirements**: Legal/regulatory requirements are non-negotiable +3. **Accessibility Issues**: User accessibility is not optional +4. **Data Loss Risks**: Anything preventing data loss is critical +5. **Explicit User Request**: If user specifically asked for something, honor it + +**In config.yml**: +```yaml +pragmatic_mode: + exemptions: + - security + - compliance + - accessibility + - data_loss_prevention +``` + +--- + +## Summary + +Pragmatic mode adds a critical YAGNI perspective to architecture reviews: + +- ✅ Prevents over-engineering +- ✅ Focuses team on current actual needs +- ✅ Identifies simpler alternatives +- ✅ Provides evidence-based decision framework +- ⚠️ Must be balanced with quality and foresight +- ⚠️ Not a blanket "no" to all abstractions + +Use pragmatic analysis to ensure architecture reviews lead to appropriate, not excessive, engineering. diff --git a/.agents/skills/architecture-review/references/review-process.md b/.agents/skills/architecture-review/references/review-process.md new file mode 100644 index 00000000..a36aea14 --- /dev/null +++ b/.agents/skills/architecture-review/references/review-process.md @@ -0,0 +1,241 @@ +# Architecture Review Process - Detailed Guide + +This document provides detailed instructions for conducting architecture reviews with all team members. + +## Table of Contents + +1. [Individual Member Review Format](#individual-member-review-format) +2. [Collaborative Discussion Process](#collaborative-discussion-process) +3. [Analysis Guidelines by Review Type](#analysis-guidelines-by-review-type) + +--- + +## Individual Member Review Format + +Each member from `.architecture/members.yml` should review from their unique perspective using this structure: + +### Review Template + +```markdown +### [Member Name] - [Title] + +**Perspective**: [Their unique viewpoint from members.yml] + +#### Key Observations +- [Observation 1 - Specific finding about the system] +- [Observation 2 - Another specific finding] +- [Continue with notable observations] + +#### Strengths +1. **[Strength Title]**: [Detailed description of what's working well and why it matters] +2. **[Strength Title]**: [Description] +3. [Continue with strengths - aim for 3-5 key strengths] + +#### Concerns +1. **[Concern Title]** (Impact: High | Medium | Low) + - **Issue**: [Clear description of what's wrong or could be improved] + - **Why it matters**: [Impact on system, users, or development] + - **Recommendation**: [Specific actionable recommendation] + +2. **[Concern Title]** (Impact: High | Medium | Low) + - **Issue**: [Description] + - **Why it matters**: [Impact] + - **Recommendation**: [Action] + +[Continue with concerns - prioritize by impact] + +#### Recommendations +1. **[Recommendation Title]** (Priority: High/Medium/Low, Effort: Small/Medium/Large) + - **What**: [What to do] + - **Why**: [Benefit or risk addressed] + - **How**: [Brief implementation approach] + +2. **[Recommendation Title]** (Priority: High/Medium/Low, Effort: Small/Medium/Large) + - [Details] + +[Continue with recommendations - 3-7 recommendations per member] +``` + +### Member-Specific Guidance + +**Systems Architect**: +- Focus on overall system coherence and architectural patterns +- Evaluate component interactions and integration points +- Assess alignment with architectural principles +- Consider long-term evolvability + +**Domain Expert**: +- Review how well architecture represents business domain +- Evaluate bounded contexts and domain model accuracy +- Check ubiquitous language usage +- Assess semantic correctness + +**Security Specialist**: +- Identify security vulnerabilities and threats +- Evaluate authentication, authorization, encryption +- Review data protection and privacy considerations +- Assess security boundaries and attack surface + +**Performance Specialist**: +- Identify performance bottlenecks +- Evaluate scalability patterns +- Review resource utilization +- Assess caching, optimization opportunities + +**Maintainability Expert**: +- Evaluate code organization and clarity +- Assess technical debt +- Review complexity and coupling +- Consider developer experience + +**AI Engineer** (if applicable): +- Review AI/ML integration patterns +- Evaluate LLM application design +- Assess agent orchestration +- Review observability and evaluation + +**Pragmatic Enforcer** (if pragmatic mode enabled): +- See [pragmatic-integration.md](./pragmatic-integration.md) for detailed process + +--- + +## Collaborative Discussion Process + +After individual reviews, simulate a discussion between members to synthesize findings. + +### Discussion Structure + +```markdown +## Collaborative Discussion + +**[Systems Architect]**: "[Opening statement about overall architecture]" + +**[Domain Expert]**: "[Response or complementary view from domain perspective]" + +**[Security Specialist]**: "[Security concerns or validation]" + +[Continue discussion flow naturally] + +### Common Ground + +Team members agree on: +1. [Consensus point 1] +2. [Consensus point 2] +3. [Consensus point 3] + +### Areas of Debate + +**Topic: [Topic Title]** +- **[Member 1]**: [Their position] +- **[Member 2]**: [Their position] +- **Resolution**: [How team resolves or agrees to disagree] + +### Priorities Established + +The team agrees on these priorities: + +**Critical (Address Immediately)**: +1. [Critical priority] +2. [Critical priority] + +**Important (Address Soon)**: +1. [Important priority] +2. [Important priority] + +**Nice-to-Have (Consider Later)**: +1. [Nice-to-have] +2. [Nice-to-have] +``` + +### Discussion Best Practices + +1. **Cross-Reference Findings**: Members should reference and build on each other's observations +2. **Resolve Conflicts**: When members disagree, discuss trade-offs and reach consensus +3. **Prioritize Together**: Collaborate to rank recommendations by urgency and impact +4. **Be Realistic**: Consider project constraints, deadlines, and team capacity +5. **Stay Constructive**: Frame concerns as improvement opportunities + +--- + +## Analysis Guidelines by Review Type + +### Version Reviews + +**Focus on**: +- Overall architecture health at this milestone +- Components and their interactions +- Patterns and consistency +- Technical debt accumulated +- ADRs implemented or needed +- Alignment with original architecture vision + +**Key Questions**: +- Is the architecture still coherent as system has evolved? +- What technical debt needs addressing before next version? +- Are architectural principles being followed? +- What risks should be mitigated? + +### Feature Reviews + +**Focus on**: +- Feature implementation approach +- Integration with existing architecture +- Data flow and state management +- Security implications +- Performance impact +- Test coverage + +**Key Questions**: +- Does this feature fit the existing architecture? +- Are there better architectural approaches? +- What are the integration risks? +- How does this impact scalability? + +### Component Reviews + +**Focus on**: +- Component architecture and structure +- Dependencies and coupling +- Boundaries and interfaces +- Responsibilities and cohesion +- Testability + +**Key Questions**: +- Is component well-designed and focused? +- Are boundaries clear and appropriate? +- Is it properly decoupled? +- Does it follow single responsibility principle? + +--- + +## Tips for High-Quality Reviews + +### Be Specific +- ❌ "The code is messy" +- ✅ "The `UserService` class has 15 methods mixing authentication, authorization, and profile management - violates single responsibility" + +### Provide Context +- ❌ "Add caching" +- ✅ "Add Redis caching for user profile queries - currently hitting DB 50+ times per page load causing 200ms delays" + +### Suggest Solutions +- ❌ "Performance is bad" +- ✅ "Batch database queries in `OrderProcessor.process()` to reduce N+1 queries - should improve processing time by 70%" + +### Balance Positive and Negative +- Don't just list problems +- Recognize what's working well +- Explain why good patterns should be sustained + +### Be Actionable +- Every concern should have a recommendation +- Recommendations should be concrete and implementable +- Estimate effort (Small/Medium/Large) and priority (High/Medium/Low) + +--- + +## Reference Documentation + +For the full review document template, see [../assets/review-template.md](../assets/review-template.md). + +For pragmatic mode integration, see [pragmatic-integration.md](./pragmatic-integration.md). diff --git a/.agents/skills/architecture-status/SKILL.md b/.agents/skills/architecture-status/SKILL.md new file mode 100644 index 00000000..fc7c7893 --- /dev/null +++ b/.agents/skills/architecture-status/SKILL.md @@ -0,0 +1,230 @@ +--- +name: architecture-status +description: Reports on the health and state of architecture documentation (counts of ADRs, reviews, activity levels, documentation gaps). Use when the user asks "What's our architecture status?", "Show architecture documentation", "How many ADRs do we have?", "What decisions are documented?", "Architecture health check", or wants an overview/summary of documentation state. Do NOT use for listing team members (use list-members), creating new documents (use create-adr), or conducting reviews (use architecture-review or specialist-review). +allowed-tools: Read,Glob,Grep +--- + +# Architecture Status + +Provides overview of architecture documentation state. + +## Process + +### 1. Check Framework Setup +If `.architecture/` doesn't exist: +``` +The AI Software Architect framework is not set up yet. + +To get started: "Setup ai-software-architect" + +Once set up, you'll have: +- Architectural Decision Records (ADRs) +- Architecture reviews +- Specialist reviews +- Recalibration tracking +``` + +### 2. Gather Information +Collect from `.architecture/`: +- **ADRs**: Count files in `decisions/adrs/`, note recent ones, check statuses +- **Reviews**: Count files in `reviews/`, categorize (version/feature/specialist/initial) +- **Recalibration**: Count files in `recalibration/`, check progress docs +- **Comparisons**: List files in `comparisons/` +- **Team**: Count members in `members.yml` +- **Last Activity**: Most recent date from any document + +### 3. Generate Status Report +```markdown +# Architecture Framework Status + +**Report Date**: [Date] +**Project**: [Project name if known] + +## Summary + +**Health Status**: Excellent | Good | Needs Attention | Inactive + +**Key Metrics**: +- ADRs: [count] +- Reviews: [count] +- Recalibration Plans: [count] +- Team Members: [count] +- Last Activity: [Date] + +## Architectural Decision Records + +**Total**: [count] + +**Recent ADRs**: +1. ADR-[XXX]: [Title] ([Status], [Date]) +2. ADR-[YYY]: [Title] ([Status], [Date]) +[List 5-10 most recent] + +**By Status**: +- ✅ Accepted: [count] +- 🔄 Proposed: [count] +- ⚠️ Deprecated: [count] +- 🔀 Superseded: [count] + +**Coverage**: [Main areas covered: Data, Security, Infrastructure, etc.] + +## Architecture Reviews + +**Total**: [count] + +**Version Reviews**: [List with dates] +**Feature Reviews**: [List with dates] +**Specialist Reviews**: [List with dates] + +**Most Recent**: [Title] ([Date]) + +## Recalibration + +**Total Documents**: [count] + +**Active**: +1. [Target]: [Status], [Completion %] + +**Status**: +- ✅ Completed: [count] +- 🔄 In Progress: [count] +- 📋 Planned: [count] + +## Architecture Team + +**Total Members**: [count] + +**Team**: [List member titles] + +**Coverage**: Security ([count]), Performance ([count]), System Design ([count]), etc. + +**View full roster**: "List architecture members" + +## Activity + +**Recent**: +- [Date]: Created ADR-XXX: [Title] +- [Date]: Completed review for [target] +- [Date]: [Activity] + +**Level**: High | Medium | Low | Inactive + +## Documentation Health + +**Completeness**: [X%] + +**Strengths**: +- ✅ [What's well documented] + +**Gaps**: +- ⚠️ [What needs attention] + +**Recommendations**: +1. [Recommendation 1] +2. [Recommendation 2] + +## Quick Actions + +**Create**: +- "Create ADR for [decision]" +- "Start architecture review for [version/feature]" +- "Ask [specialist] to review [target]" + +**View**: +- "List architecture members" +- [Specific docs to review based on status] + +**Update**: +- [Actions based on current state] + +--- +``` + +### 4. Analyze Health +**Health indicators**: +- **Excellent**: Regular ADRs, recent reviews, active recalibration +- **Good**: Some ADRs, occasional reviews +- **Needs Attention**: Old docs, no recent activity +- **Inactive**: Framework unused + +### 5. Provide Recommendations +**Based on status**: + +**If well-maintained**: +``` +✅ Excellent documentation discipline! + +Keep momentum: +- Continue documenting decisions +- Regular reviews (quarterly/before releases) +- Track recalibration progress +``` + +**If partially used**: +``` +⚠️ Good foundations, room for improvement. + +Suggestions: +- Document 3-5 key decisions as ADRs +- Schedule architecture review +- Address review findings +``` + +**If minimal usage**: +``` +❌ Framework underutilized. + +Get started: +1. Document your most important decisions as ADRs +2. Conduct initial architecture review +3. Make documentation a regular habit +``` + +### 6. Make It Actionable +Always end with: +- Specific commands to run +- Concrete actions to improve +- Examples relevant to their status + +## Metrics to Track + +**Volume**: Total ADRs, reviews, recalibration docs, team members +**Activity**: Last update, docs per month, active vs completed +**Coverage**: Decision areas, review types, specialist expertise +**Health**: Documentation completeness, review frequency, progress tracking + +## Error Handling +- No `.architecture/`: Offer setup +- Permission issues: Report and suggest fixes +- Corrupted files: Note which have issues +- Empty directories: Suggest starting points + +## Related Skills + +**Based on Status Results**: + +**If Documentation Gaps Found**: +- "Create ADR for [missing decision]" - Fill documentation gaps +- "Start architecture review for [area]" - Comprehensive assessment +- "Ask [specialist] to review [weak area]" - Focused improvement + +**If Status is Good**: +- "List architecture members" - See your active team +- Continue regular documentation practices +- Schedule periodic reviews + +**Regular Workflow**: +1. Start work → "What's our architecture status?" → Identify gaps +2. Make changes → Document with ADRs → Status check to verify +3. Before release → Status check → Architecture review → Address findings + +**Workflow Examples**: +1. Status check → Find 0 ADRs → Create ADRs for key decisions → Status check shows progress +2. Status check → See old reviews → Request new architecture review → Update status +3. Weekly: Status check → Track documentation health → Maintain good practices + +## Notes +- Be honest but encouraging +- Focus on actionable next steps +- Show value of maintained documentation +- Adapt tone to current state diff --git a/.agents/skills/create-adr/SKILL.md b/.agents/skills/create-adr/SKILL.md new file mode 100644 index 00000000..b426ca35 --- /dev/null +++ b/.agents/skills/create-adr/SKILL.md @@ -0,0 +1,122 @@ +--- +name: create-adr +description: Creates a NEW Architectural Decision Record (ADR) documenting a specific architectural decision. Use when the user requests "Create ADR for [topic]", "Document decision about [topic]", "Write ADR for [choice]", or when documenting technology choices, patterns, or architectural approaches. Do NOT use for reviews (use architecture-review or specialist-review), checking existing ADRs (use architecture-status), or general documentation. +allowed-tools: Read,Write,Bash(ls:*,grep:*) +--- + +# Create Architectural Decision Record (ADR) + +Creates structured ADRs following the framework's template. + +## Process + +### 1. Gather Context +Ask if needed: +- What decision is being made? +- What problem does it solve? +- What alternatives were considered? +- What are the trade-offs? + +### 2. Generate ADR Number +```bash +# Find highest ADR number +ls .architecture/decisions/adrs/ | grep -E "^ADR-[0-9]+" | sed 's/ADR-//' | sed 's/-.*//' | sort -n | tail -1 +``` +New ADR = next sequential number (e.g., if highest is 003, create 004) + +### 3. Validate and Sanitize Input +**Security**: Sanitize user input to prevent path traversal and injection: +- Remove or replace: `..`, `/`, `\`, null bytes, control characters +- Convert to lowercase kebab-case: spaces → hyphens, remove special chars +- Limit length: max 80 characters for filename portion +- Validate result: ensure filename contains only [a-z0-9-] + +### 4. Create Filename +Format: `ADR-XXX-kebab-case-title.md` + +Examples: +- `ADR-001-use-react-for-frontend.md` +- `ADR-002-choose-postgresql-database.md` + +**Valid input**: "Use React for Frontend" → `use-react-for-frontend` +**Invalid blocked**: "../etc/passwd" → sanitized or rejected + +### 5. Check Configuration +- Read `.architecture/config.yml` to check if pragmatic_mode is enabled +- If enabled and applies to ADR creation, include Pragmatic Enforcer analysis + +### 6. Write ADR +Use the template from `.architecture/templates/adr-template.md`: + +**Core sections**: +- Status, Context, Decision Drivers, Decision, Consequences +- Implementation, Alternatives Considered, Validation, References + +**If pragmatic_mode is enabled**: Add Pragmatic Enforcer Analysis section: +- Necessity Assessment (0-10): Current need, future need, cost of waiting, evidence +- Complexity Assessment (0-10): Added complexity, maintenance, learning curve, dependencies +- Alternative Analysis: Review if simpler alternatives adequately considered +- Simpler Alternative Proposal: Concrete proposal for simpler approach +- Recommendation: Approve / Approve with simplifications / Defer / Recommend against +- Pragmatic Score: Necessity, Complexity, Ratio (target <1.5) +- Overall Assessment: Appropriate engineering vs over-engineering + +**If deferrals enabled**: Track deferred decisions in `.architecture/deferrals.md` + +### 7. Save ADR +Write to: `.architecture/decisions/adrs/ADR-XXX-title.md` + +### 8. Report to User +``` +Created ADR-XXX: [Title] + +Location: .architecture/decisions/adrs/ADR-XXX-title.md +Status: [Status] + +Key Points: +- Decision: [Summary] +- Main benefit: [Key benefit] +- Main trade-off: [Key trade-off] + +Next Steps: +- [Immediate action 1] +- [Immediate action 2] +``` + +## When to Create ADRs +**Do create for**: +- Technology choices (frameworks, databases, languages) +- Architectural patterns (microservices, event-driven, etc.) +- Infrastructure decisions (cloud provider, deployment) +- Security approaches (authentication, encryption) + +**Don't create for**: +- Implementation details (function names, variable names) +- Temporary decisions +- Minor decisions with limited impact + +## Status Lifecycle +- **Proposed**: Documented but not approved +- **Accepted**: Approved and should be implemented +- **Deprecated**: No longer best practice +- **Superseded**: Replaced by newer ADR (reference it) + +## Related Skills + +**Before Creating ADR**: +- "What's our architecture status?" - Check existing ADRs to avoid duplication +- "List architecture members" - See who should review the decision + +**After Creating ADR**: +- "Ask [specialist] to review [the ADR]" - Get focused expert review +- "Start architecture review for [version]" - Include in comprehensive review + +**Workflow Examples**: +1. Create ADR → Ask Security Specialist to review → Revise ADR +2. Architecture review → Create ADRs for key decisions → Status check + +## Notes +- Focus on "why" more than "what" +- Be honest about trade-offs +- Keep it concise but complete +- ADRs can be updated as new information emerges diff --git a/.agents/skills/list-members/SKILL.md b/.agents/skills/list-members/SKILL.md new file mode 100644 index 00000000..92f390fa --- /dev/null +++ b/.agents/skills/list-members/SKILL.md @@ -0,0 +1,175 @@ +--- +name: list-members +description: Displays the roster of architecture team members with their specialties and expertise areas. Use when the user asks "Who's on the architecture team?", "List architecture members", "Show me the architects", "What specialists are available?", "Who can I ask for reviews?", or wants to discover available experts. Do NOT use for requesting reviews (use specialist-review or architecture-review) or checking documentation status (use architecture-status). +allowed-tools: Read +--- + +# List Architecture Members + +Displays all architecture team members and their expertise areas. + +## Process + +### 1. Check Setup +If `.architecture/members.yml` doesn't exist: +``` +The AI Software Architect framework hasn't been set up yet. + +To get started: "Setup ai-software-architect" +``` + +### 2. Load Members +Read `.architecture/members.yml` and parse all members (id, name, title, specialties, disciplines, skillsets, domains, perspective). + +### 3. Display Team Roster +```markdown +# Architecture Team Members + +Your AI Software Architect team consists of [count] specialized reviewers. + +Total Members: [count] + +--- + +## Team Roster + +### [Member 1 Name] - [Member 1 Title] + +**ID**: `[member_id]` + +**Specialties**: [Specialty 1], [Specialty 2], [Specialty 3] + +**Disciplines**: [Discipline 1], [Discipline 2] + +**Domains**: [Domain 1], [Domain 2], [Domain 3] + +**Perspective**: [Their unique perspective] + +**Request review**: `Ask [Member Title] to review [your target]` + +--- + +[Repeat for all members] + +--- + +## Quick Reference + +**Specialist reviews**: +- `Ask [Specialist Title] to review [target]` + +**Examples**: +- "Ask Security Specialist to review authentication" +- "Ask Performance Specialist to review database queries" +- "Ask [Your Specialist] to review [anything]" + +**Full architecture review**: +- `Start architecture review for version X.Y.Z` + +**Other commands**: +- `Create ADR for [decision topic]` +- `What's our architecture status?` + +--- + +## Team by Specialty + +[Group members by their primary domains/specialties] + +**Security & Compliance**: [Members] +**Performance & Scalability**: [Members] +**Code Quality & Maintainability**: [Members] +**Domain & Business Logic**: [Members] +**System Design & Architecture**: [Members] +**Technology-Specific**: [Members] + +--- + +## Adding New Members + +Request a review from any specialist, even if they don't exist: +- "Ask Ruby Expert to review my modules" +- "Have Accessibility Expert review forms" + +I'll create the specialist and add them to your team automatically. + +Or manually edit `.architecture/members.yml` and add: +```yaml +- id: your_specialist_id + name: "[Name]" + title: "[Title]" + specialties: ["[Specialty 1]", "[Specialty 2]"] + disciplines: ["[Discipline 1]", "[Discipline 2]"] + skillsets: ["[Skill 1]", "[Skill 2]"] + domains: ["[Domain 1]", "[Domain 2]"] + perspective: "[Brief description]" +``` + +--- + +## Using the Team + +**For focused reviews** (specific expertise): +``` +Ask [Specialist] to review [target] +``` +Fast turnaround, targeted insights + +**For comprehensive reviews** (all perspectives): +``` +Start architecture review for [version/feature] +``` +All members review, collaborative discussion + +**For decisions**: +``` +Create ADR for [decision topic] +``` +Document decisions with team input + +--- +``` + +### 4. Provide Context +After listing: +- Explain when to use specialist vs full reviews +- Show how to add new members dynamically +- Provide usage examples +- Suggest next steps + +## Example Output Summary +After showing full roster, provide a concise summary: +``` +Ready to use your architecture team: +- [N] specialists available +- Request reviews: "Ask [specialist] to review [target]" +- Add new specialists: Just ask for them by name +- Full review: "Start architecture review for [scope]" +``` + +## Error Handling +- No `.architecture/`: Offer setup instructions +- Empty `members.yml`: Show default team and offer setup +- Malformed YAML: Show what's parseable, note issues + +## Related Skills + +**After Listing Members**: +- "Ask [specialist] to review [target]" - Request focused expert review +- "Start architecture review for [scope]" - Comprehensive review with all members +- "What's our architecture status?" - See how team has been used + +**When Adding Members**: +- Just request a specialist that doesn't exist - they'll be created automatically +- "Setup ai-software-architect" - Adds members based on your tech stack + +**Workflow Examples**: +1. List members → Identify relevant specialist → Request review +2. Need new expertise → Request it → Specialist auto-created → List to verify +3. Status check → List members → Review with specific specialists + +## Notes +- Keep presentation clear and scannable +- Make it actionable with specific commands +- Encourage exploration and usage +- Explain that team can grow dynamically diff --git a/.agents/skills/pragmatic-guard/SKILL.md b/.agents/skills/pragmatic-guard/SKILL.md new file mode 100644 index 00000000..66c1ef34 --- /dev/null +++ b/.agents/skills/pragmatic-guard/SKILL.md @@ -0,0 +1,152 @@ +--- +name: pragmatic-guard +description: Enables and configures Pragmatic Guard Mode (YAGNI Enforcement) to prevent over-engineering. Use when the user requests "Enable pragmatic mode", "Turn on YAGNI enforcement", "Activate simplicity guard", "Challenge complexity", or similar phrases. +allowed-tools: Read,Edit +--- + +# Pragmatic Guard Mode + +Enables and configures the Pragmatic Guard Mode to actively challenge over-engineering. + +## Process + +### 1. Check Current Configuration +- Read `.architecture/config.yml` to check current settings +- If config.yml doesn't exist, offer to create it from `.architecture/templates/config.yml` +- Check if `pragmatic_mode.enabled` is true +- Note the intensity level (strict/balanced/lenient) +- Review exemption categories and triggers + +### 2. Enable Pragmatic Mode (if requested) +If user wants to enable: +- If config.yml doesn't exist: + ```bash + cp .architecture/templates/config.yml .architecture/config.yml + ``` +- Set `pragmatic_mode.enabled: true` +- Confirm intensity level with user or use default (balanced) +- Create `.architecture/deferrals.md` from template if it doesn't exist: + ```bash + cp .architecture/templates/deferrals.md .architecture/deferrals.md + ``` +- Inform user about mode activation and current settings + +### 3. Configure Intensity Level +Ask user which intensity they prefer if not specified: + +**Strict Mode**: +- Challenges aggressively, requires strong justification for any complexity +- Questions every "should" and "could" +- Pushes for absolute minimal implementation +- Best for: New projects, MVPs, startups with limited resources + +**Balanced Mode** (RECOMMENDED): +- Challenges thoughtfully, accepts justified complexity +- Seeks middle ground between simplicity and best practices +- Questions "should" but accepts reasonable "must" +- Best for: Most projects, growing teams, moderate complexity + +**Lenient Mode**: +- Raises concerns without blocking +- Suggests simpler alternatives as options +- Focuses on major complexity additions only +- Best for: Established projects, teams with strong architecture practices + +### 4. Configure Triggers (optional) +Ask if user wants to customize which situations trigger pragmatic analysis: +- New abstraction layers (interfaces, base classes, DI containers) +- New external dependencies (libraries, frameworks, services) +- New architectural patterns (repository, strategy, observer) +- Scope expansion beyond initial requirements +- Performance optimizations without evidence +- Test infrastructure upfront +- Flexibility/configurability additions + +### 5. Configure Exemptions (optional) +Confirm exemption categories where best practices should be maintained: +- Security-critical features (always exempt by default) +- Data integrity (always exempt by default) +- Compliance requirements (always exempt by default) +- Accessibility (always exempt by default) + +### 6. Understanding the Pragmatic Enforcer +Explain how the Pragmatic Enforcer will participate: + +**In Architecture Reviews**: +- Reviews each architect's recommendations +- Applies necessity assessment (0-10 scoring) +- Applies complexity assessment (0-10 scoring) +- Proposes simpler alternatives +- Calculates pragmatic score (complexity/necessity ratio, target <1.5) + +**In ADR Creation**: +- Challenges proposed decisions +- Questions whether complexity is justified by current needs +- Suggests phased approaches or deferrals +- Documents trigger conditions for deferred decisions + +**Question Framework**: +- **Necessity**: "Do we need this right now?" "What breaks without it?" +- **Simplicity**: "What's the simplest thing that could work?" +- **Cost**: "What's the cost of implementing now vs waiting?" +- **Alternatives**: "What if we just...?" "Could we use existing tools?" +- **Best Practices**: "Does this best practice apply to our context?" + +### 7. Deferrals Tracking +If `behavior.track_deferrals` is true: +- Maintain `.architecture/deferrals.md` with deferred decisions +- Include trigger conditions for each deferral +- Track when deferrals are implemented or remain deferred + +### 8. Report to User +``` +Pragmatic Guard Mode: [Enabled | Disabled] + +Configuration: +- Intensity: [strict | balanced | lenient] +- Apply to: [List of phases where it applies] +- Deferrals tracking: [enabled | disabled] + +Exemptions: +- Security-critical: [enabled | disabled] +- Data integrity: [enabled | disabled] +- Compliance: [enabled | disabled] +- Accessibility: [enabled | disabled] + +Triggers: +[List of active triggers] + +The Pragmatic Enforcer will now: +- Challenge recommendations from other architects +- Question complexity and abstractions +- Propose simpler alternatives +- Calculate pragmatic scores (target ratio <1.5) +- [If enabled] Track deferred decisions in .architecture/deferrals.md + +Next Steps: +- Create an ADR: "Create ADR for [topic]" (will include pragmatic analysis) +- Start a review: "Start architecture review for [target]" (will include pragmatic challenges) +- Adjust settings: Edit .architecture/config.yml +``` + +## When to Use Pragmatic Mode + +**Enable for**: +- New projects or MVPs +- Teams prone to over-engineering +- Resource-constrained environments +- Learning environments (teaching YAGNI) +- Projects with tight deadlines + +**Consider disabling for**: +- Mature systems with established patterns +- High-complexity domains requiring abstractions +- Teams already practicing strong YAGNI principles +- Projects with specific architectural requirements + +## Notes +- Pragmatic mode is about finding balance, not blocking progress +- The Pragmatic Enforcer's role is to question and challenge, not to veto +- Intensity level should match team maturity and project complexity +- Exemptions ensure critical areas maintain appropriate rigor +- Deferrals can always be implemented when triggered by actual needs diff --git a/.agents/skills/setup-architect/SKILL.md b/.agents/skills/setup-architect/SKILL.md new file mode 100644 index 00000000..d72a83eb --- /dev/null +++ b/.agents/skills/setup-architect/SKILL.md @@ -0,0 +1,195 @@ +--- +name: setup-architect +description: Sets up and installs the AI Software Architect framework in a NEW project for the FIRST time. Use when the user requests "Setup .architecture", "Setup ai-software-architect", "Initialize architecture framework", "Install software architect", or similar setup/installation phrases. Do NOT use for checking status (use architecture-status), creating documents (use create-adr or reviews), or when framework is already set up. +allowed-tools: Read,Write,Edit,Glob,Grep,Bash +--- + +# Setup AI Software Architect Framework + +Sets up and customizes the AI Software Architect framework for a project. + +## Overview + +This skill performs a complete framework installation: +1. Verifies prerequisites (framework cloned, project root confirmed) +2. Analyzes project (languages, frameworks, structure, patterns) +3. Installs framework files and directory structure +4. Customizes team members and principles for detected tech stack +5. Performs initial system analysis +6. Reports customizations and findings + +**Detailed procedures**: [references/installation-procedures.md](references/installation-procedures.md) +**Customization guide**: [references/customization-guide.md](references/customization-guide.md) + +## High-Level Workflow + +### 1. Verify Prerequisites + +Check requirements before installation: +- `.architecture/.architecture/` directory exists (cloned framework) +- Currently in project root directory + +**If missing**: Guide user to clone framework first. + +### 2. Analyze Project + +Identify project characteristics: +- **Languages**: JavaScript/TypeScript, Python, Ruby, Java, Go, Rust +- **Frameworks**: React, Vue, Django, Rails, Spring, etc. +- **Infrastructure**: Testing setup, CI/CD, package managers +- **Structure**: Directory layout, architectural patterns + +Use `Glob` and `Grep` to detect technologies, `Read` to examine configs. + +### 3. Install Framework + +Execute installation steps (see [references/installation-procedures.md](references/installation-procedures.md)): +- Copy framework files to `.architecture/` +- Remove clone directory +- Create directory structure (decisions/adrs, reviews, recalibration, etc.) +- Initialize configuration from templates +- Set up agent documentation (ADR-006 progressive disclosure) + +**Critical**: Follow safety procedures when removing `.git/` directory. + +### 4. Customize Architecture Team + +Add technology-specific members to `.architecture/members.yml`: +- **JavaScript/TypeScript**: JavaScript Expert, framework specialists (React/Vue/Angular) +- **Python**: Python Expert, framework specialists (Django/Flask/FastAPI) +- **Ruby**: Ruby Expert, Rails Architect +- **Java**: Java Expert, Spring Boot Specialist +- **Go**: Go Expert, Microservices Architect +- **Rust**: Rust Expert, Systems Programmer + +Use template from [assets/member-template.yml](assets/member-template.yml). + +**Keep core members**: Systems Architect, Domain Expert, Security, Performance, Maintainability, AI Engineer, Pragmatic Enforcer. + +**Customization details**: [references/customization-guide.md § Customize Team Members](references/customization-guide.md#customize-architecture-team-members) + +### 5. Customize Architectural Principles + +Add framework-specific principles to `.architecture/principles.md`: +- **React**: Component composition, hooks, unidirectional data flow +- **Rails**: Convention over configuration, DRY, RESTful design +- **Django**: Explicit over implicit, reusable apps, use built-ins + +**Principle examples**: [references/customization-guide.md § Customize Principles](references/customization-guide.md#customize-architectural-principles) + +### 6. Update CLAUDE.md Integration + +If `CLAUDE.md` exists in project root, append framework usage section: +- Available commands +- Where to find documentation +- How to invoke skills + +**Template**: [references/customization-guide.md § Update CLAUDE.md](references/customization-guide.md#update-claudemd-integration) + +### 7. Cleanup + +Remove framework development files: +- Framework documentation (README.md, USAGE*.md, INSTALL.md) +- Template `.git/` directory (with **critical safety checks**) + +**⚠️ IMPORTANT**: Follow all safeguards in [references/installation-procedures.md § Cleanup](references/installation-procedures.md#cleanup-procedures). + +### 8. Create Initial System Analysis + +Generate comprehensive initial analysis document: +- Each member analyzes system from their perspective +- System overview (stack, structure, patterns) +- Strengths identified +- Concerns raised (with impact levels) +- Recommendations prioritized (Critical/Important/Nice-to-Have) +- Collaborative synthesis of findings + +Save to `.architecture/reviews/initial-system-analysis.md`. + +**Template**: [assets/initial-analysis-template.md](assets/initial-analysis-template.md) + +### 9. Report to User + +Provide setup summary: + +``` +AI Software Architect Framework Setup Complete + +Customizations: +- Added [N] technology specialists: [list] +- Customized principles for: [frameworks] +- Configuration: Pragmatic mode [enabled/disabled] + +Initial Analysis Highlights: +- Overall assessment: [assessment] +- Top strength: [strength] +- Top concern: [concern] +- Critical recommendation: [recommendation] + +Location: .architecture/reviews/initial-system-analysis.md + +Next Steps: +- Review initial analysis findings +- "List architecture members" to see customized team +- "Create ADR for [first decision]" to start documenting +- "What's our architecture status?" to verify setup +``` + +## Error Handling + +**Framework not cloned**: +``` +The framework must be cloned first. Please run: + +git clone https://github.com/codenamev/ai-software-architect .architecture/.architecture + +Then run setup again. +``` + +**Already set up**: +``` +Framework appears to be already set up. + +To verify: "What's our architecture status?" +To reconfigure: Manually edit .architecture/members.yml and .architecture/principles.md +``` + +**Unclear project structure**: +``` +Could not clearly identify project type. Please describe: +- Primary programming language(s) +- Framework(s) used +- Project purpose + +I'll customize the framework accordingly. +``` + +## Related Skills + +**After Setup**: +- `list-members` - View customized team +- `architecture-status` - Verify setup completion +- `create-adr` - Document first decision + +**Initial Work**: +- Review `initial-system-analysis.md` findings +- `specialist-review` - Deep-dive on specific concerns +- `create-adr` - Document existing key decisions + +**Workflow Example**: +Setup → Review initial analysis → Create ADRs → Status check → Regular reviews + +## Notes + +- Customize based on **actual** project, not every possible option +- Be specific about **why** each customization was made +- Initial analysis should be thorough but focused on actionable findings +- Safety checks during cleanup are **non-negotiable** + +## Documentation + +- **Installation details**: [references/installation-procedures.md](references/installation-procedures.md) +- **Customization guide**: [references/customization-guide.md](references/customization-guide.md) +- **Initial analysis template**: [assets/initial-analysis-template.md](assets/initial-analysis-template.md) +- **Member template**: [assets/member-template.yml](assets/member-template.yml) +- **Common patterns**: [../_patterns.md](../_patterns.md) diff --git a/.agents/skills/setup-architect/assets/initial-analysis-template.md b/.agents/skills/setup-architect/assets/initial-analysis-template.md new file mode 100644 index 00000000..ab27793d --- /dev/null +++ b/.agents/skills/setup-architect/assets/initial-analysis-template.md @@ -0,0 +1,400 @@ +# Initial System Analysis + +**Project**: [Project Name] +**Date**: [YYYY-MM-DD] +**Analysis Type**: Initial Setup Assessment +**Analysts**: [List all architecture members participating] + +--- + +## Executive Summary + +[Write 2-3 paragraphs providing a high-level overview of the system being analyzed. Include: +- What the system does (purpose and domain) +- Primary technologies and stack +- Current state and maturity level +- Overall architectural health +] + +**Overall Assessment**: [Excellent | Good | Adequate | Needs Attention] + +**Key Findings**: +- [Major finding 1] +- [Major finding 2] +- [Major finding 3] + +**Critical Recommendations**: +- [Top priority recommendation] +- [Second priority recommendation] + +--- + +## System Overview + +### Project Information + +**Primary Language(s)**: [e.g., JavaScript/TypeScript, Python, Ruby] + +**Frameworks**: [e.g., React, Rails, Django, Express] + +**Architecture Style**: [e.g., Monolith, Microservices, Serverless, Hybrid] + +**Deployment**: [e.g., Heroku, AWS, Docker, Kubernetes] + +**Team Size**: [Number of developers] + +**Project Age**: [e.g., 2 years, New project, Legacy system] + +### Technology Stack + +**Frontend**: +- [Technology 1] +- [Technology 2] + +**Backend**: +- [Technology 1] +- [Technology 2] + +**Database**: +- [Primary database] +- [Cache layer if any] + +**Infrastructure**: +- [Hosting/cloud provider] +- [CI/CD tools] +- [Monitoring/observability] + +### Project Structure + +``` +[Provide high-level directory structure] +/ +├── src/ +│ ├── components/ +│ ├── services/ +│ └── ... +├── tests/ +└── ... +``` + +**Key Observations**: +- [Structure observation 1] +- [Structure observation 2] + +--- + +## Individual Member Analyses + +### [Member Name] - [Title] + +**Perspective**: [Their specialized viewpoint] + +#### Current State Assessment + +[Detailed analysis from this member's perspective. Include: +- What they examined +- How they evaluated it +- What they discovered +] + +#### Strengths Identified + +1. **[Strength]**: [Why this is valuable and how it benefits the project] + +2. **[Strength]**: [Description] + +3. **[Strength]**: [Description] + +#### Concerns Raised + +1. **[Concern]** (Impact: High | Medium | Low) + - **Issue**: [What's problematic] + - **Why It Matters**: [Impact on project/team] + - **Recommendation**: [Suggested action] + +2. **[Concern]** (Impact: High | Medium | Low) + - **Issue**: [Description] + - **Why It Matters**: [Impact] + - **Recommendation**: [Action] + +#### Initial Recommendations + +1. **[Recommendation]** (Priority: Critical | Important | Nice-to-Have, Effort: Small | Medium | Large) + - **What**: [Action to take] + - **Why**: [Benefit or risk addressed] + - **How**: [Implementation approach] + +2. **[Recommendation]** (Priority, Effort) + - [Details] + +[Repeat for each architecture member] + +--- + +## Collaborative Synthesis + +[After individual analyses, synthesize findings across all members] + +### Common Themes + +**Strengths** (What multiple members praised): +1. [Common strength 1] +2. [Common strength 2] + +**Concerns** (What multiple members flagged): +1. [Common concern 1] +2. [Common concern 2] + +**Disagreements** (Where members had different views): +- **Topic**: [Topic of disagreement] + - **[Member 1]**: [Their view] + - **[Member 2]**: [Their view] + - **Resolution**: [How to reconcile or way forward] + +### Prioritized Findings + +**Critical (Address Immediately)**: +1. **[Finding]**: [Why critical, impact if not addressed] +2. **[Finding]**: [Details] + +**Important (Address in Near Term)**: +1. **[Finding]**: [Why important] +2. **[Finding]**: [Details] + +**Nice-to-Have (Consider for Future)**: +1. **[Finding]**: [Why beneficial but not urgent] +2. **[Finding]**: [Details] + +--- + +## Architectural Health Assessment + +### Code Quality + +**Rating**: [1-10] + +**Observations**: +- [Observation about code organization] +- [Observation about code clarity] +- [Observation about technical debt] + +**Key Issues**: +- [Issue 1] +- [Issue 2] + +### Testing + +**Coverage**: [Percentage if known, or qualitative assessment] + +**Rating**: [1-10] + +**Observations**: +- [Test presence and quality] +- [Test types (unit, integration, e2e)] +- [Test infrastructure] + +**Gaps**: +- [Gap 1] +- [Gap 2] + +### Documentation + +**Rating**: [1-10] + +**Observations**: +- [README quality] +- [API documentation] +- [Architecture documentation] +- [Code comments] + +**Missing**: +- [What documentation is missing] + +### Security + +**Rating**: [1-10] + +**Observations**: +- [Authentication/authorization approach] +- [Data protection measures] +- [Known vulnerabilities] + +**Concerns**: +- [Security concern 1] +- [Security concern 2] + +### Performance + +**Rating**: [1-10] + +**Observations**: +- [Performance characteristics] +- [Scalability considerations] +- [Known bottlenecks] + +**Concerns**: +- [Performance concern 1] +- [Performance concern 2] + +### Maintainability + +**Rating**: [1-10] + +**Observations**: +- [How easy is it to change?] +- [Developer onboarding experience] +- [Build/deploy process] + +**Challenges**: +- [Maintainability challenge 1] +- [Maintainability challenge 2] + +--- + +## Technical Debt Inventory + +### High Priority Debt + +1. **[Debt Item]** + - **Impact**: [How it affects development] + - **Effort to Resolve**: [Small | Medium | Large] + - **Recommendation**: [When and how to address] + +2. **[Debt Item]**: [Details] + +### Medium Priority Debt + +1. **[Debt Item]**: [Details] +2. **[Debt Item]**: [Details] + +### Low Priority Debt + +1. **[Debt Item]**: [Details] + +--- + +## Risk Assessment + +### Technical Risks + +1. **[Risk]** (Likelihood: High/Medium/Low, Impact: High/Medium/Low) + - **Description**: [What could go wrong] + - **Impact**: [Consequences if it happens] + - **Mitigation**: [How to reduce or eliminate] + +2. **[Risk]** (Likelihood, Impact) + - [Details] + +### Business Risks + +1. **[Risk]** (Likelihood, Impact) + - [Details] + +### Operational Risks + +1. **[Risk]** (Likelihood, Impact) + - [Details] + +--- + +## Recommendations + +### Immediate Actions (0-2 Weeks) + +1. **[Action]** + - **Why**: [Problem being solved] + - **How**: [Implementation approach] + - **Owner**: [Who should do this] + - **Success Criteria**: [How to know it's done] + - **Effort**: [Time estimate] + +2. **[Action]**: [Details] + +### Short-Term Actions (2-8 Weeks) + +1. **[Action]** + - **Why**: [Benefit] + - **How**: [Approach] + - **Owner**: [Responsible party] + - **Success Criteria**: [Completion criteria] + - **Effort**: [Estimate] + +2. **[Action]**: [Details] + +### Long-Term Initiatives (2-6 Months) + +1. **[Initiative]** + - **Why**: [Strategic value] + - **How**: [High-level roadmap] + - **Owner**: [Responsible party] + - **Success Criteria**: [Long-term goals] + - **Effort**: [Estimate] + +2. **[Initiative]**: [Details] + +--- + +## Success Metrics + +Define measurable targets to track improvement: + +1. **[Metric Name]** + - **Baseline**: [Current value] + - **Target**: [Desired value] + - **Timeline**: [When to achieve] + - **How to Measure**: [Measurement method] + +2. **[Metric]**: Baseline → Target (Timeline) + +3. **[Metric]**: Baseline → Target (Timeline) + +[Examples: +- Test Coverage: 30% → 70% (3 months) +- Build Time: 10 min → 3 min (6 weeks) +- Documentation Score: 4/10 → 8/10 (2 months) +] + +--- + +## Suggested Next Steps + +Based on this initial analysis: + +1. **[Step 1]**: [Specific actionable next step] +2. **[Step 2]**: [Next step] +3. **[Step 3]**: [Next step] + +**Documentation**: +- Create ADRs for key architectural decisions identified +- Document existing patterns in ADRs +- Schedule regular architecture reviews + +**Process**: +- Establish review cadence (quarterly recommended) +- Define ADR creation criteria +- Set up architecture recalibration process + +--- + +## Appendix + +### Analysis Methodology + +This analysis was conducted using the AI Software Architect framework. Each member analyzed the system from their specialized perspective, then collaborated to synthesize findings and prioritize recommendations. + +**Members Participating**: +- [Member 1] - [Role] +- [Member 2] - [Role] +- [etc.] + +### Glossary + +[Define project-specific terms or acronyms used in analysis] + +- **[Term]**: [Definition] +- **[Acronym]**: [Expansion and meaning] + +--- + +**Analysis Complete** +**Next Review**: [Suggested date - typically 3-6 months after setup] diff --git a/.agents/skills/setup-architect/assets/member-template.yml b/.agents/skills/setup-architect/assets/member-template.yml new file mode 100644 index 00000000..4ea0c891 --- /dev/null +++ b/.agents/skills/setup-architect/assets/member-template.yml @@ -0,0 +1,138 @@ +# Architecture Team Member Template +# +# Use this template to add new team members to .architecture/members.yml +# +# Copy this entire member block and customize for your technology stack + +- id: [unique_identifier] # lowercase with underscores, e.g., javascript_expert + name: "[Full Name]" # Display name, e.g., "Alex Rivera" + title: "[Role Title]" # Professional title, e.g., "JavaScript Expert" + + # Specialties: Main areas of expertise (3-5 items) + specialties: + - "[Specialty 1]" # e.g., "Modern JavaScript/TypeScript" + - "[Specialty 2]" # e.g., "Frontend architecture" + - "[Specialty 3]" # e.g., "Build tools" + + # Disciplines: Professional disciplines and practices (3-5 items) + disciplines: + - "[Discipline 1]" # e.g., "Clean code" + - "[Discipline 2]" # e.g., "Performance optimization" + - "[Discipline 3]" # e.g., "Test-driven development" + + # Skillsets: Specific technical skills (3-5 items) + skillsets: + - "[Skill 1]" # e.g., "ES6+" + - "[Skill 2]" # e.g., "Async programming" + - "[Skill 3]" # e.g., "Module systems" + + # Domains: Application domains and contexts (3-5 items) + domains: + - "[Domain 1]" # e.g., "Web applications" + - "[Domain 2]" # e.g., "API services" + - "[Domain 3]" # e.g., "Cloud infrastructure" + + # Perspective: Unique viewpoint in 1-2 sentences + perspective: "[Description of how this member approaches architecture reviews and what lens they bring]" + +# ============================================================================ +# EXAMPLES BY TECHNOLOGY STACK +# ============================================================================ + +# JavaScript/TypeScript Example: +# - id: javascript_expert +# name: "Alex Rivera" +# title: "JavaScript Expert" +# specialties: ["Modern JavaScript/TypeScript", "Frontend architecture", "Build tools"] +# disciplines: ["Clean code", "Performance", "Cross-browser compatibility"] +# skillsets: ["ES6+", "Async programming", "Module systems"] +# domains: ["Web applications", "Node.js backends", "Build pipelines"] +# perspective: "Focuses on JavaScript best practices and modern language features" + +# Python Example: +# - id: python_expert +# name: "Dr. Sarah Chen" +# title: "Python Expert" +# specialties: ["Python best practices", "Package architecture", "Type hints"] +# disciplines: ["Pythonic code", "Testing patterns", "Performance"] +# skillsets: ["Modern Python (3.10+)", "Async/await", "Data structures"] +# domains: ["Backend services", "Data processing", "APIs"] +# perspective: "Advocates for clean, Pythonic solutions following PEP guidelines" + +# Ruby Example: +# - id: ruby_expert +# name: "Marcus Johnson" +# title: "Ruby Expert" +# specialties: ["Ruby idioms", "Metaprogramming", "Rails patterns"] +# disciplines: ["Convention over configuration", "DRY", "Expressive code"] +# skillsets: ["Ruby 3.x", "Rails architecture", "Gem development"] +# domains: ["Web applications", "API services", "Background jobs"] +# perspective: "Emphasizes Ruby's elegance and Rails conventions" + +# Go Example: +# - id: go_expert +# name: "Dmitri Ivanov" +# title: "Go Expert" +# specialties: ["Idiomatic Go", "Concurrency", "Service architecture"] +# disciplines: ["Simplicity", "Composition", "Clear error handling"] +# skillsets: ["Goroutines/channels", "Standard library", "Performance"] +# domains: ["Microservices", "Cloud-native", "CLI tools"] +# perspective: "Advocates for Go's simplicity and effective concurrency" + +# Rust Example: +# - id: rust_expert +# name: "Katya Volkov" +# title: "Rust Expert" +# specialties: ["Memory safety", "Zero-cost abstractions", "Systems programming"] +# disciplines: ["Ownership/borrowing", "Type-driven design", "Performance"] +# skillsets: ["Rust idioms", "Async Rust", "Unsafe code"] +# domains: ["Systems software", "Performance-critical code", "Embedded"] +# perspective: "Emphasizes Rust's safety guarantees and performance" + +# React Specialist Example: +# - id: react_specialist +# name: "Emma Rodriguez" +# title: "React Specialist" +# specialties: ["React patterns", "Component architecture", "State management"] +# disciplines: ["Hooks", "Performance optimization", "Accessibility"] +# skillsets: ["React 18+", "Context/Redux", "Testing Library"] +# domains: ["SPAs", "Component libraries", "Web apps"] +# perspective: "Focuses on modern React patterns and component reusability" + +# Django Specialist Example: +# - id: django_specialist +# name: "Rajesh Patel" +# title: "Django Specialist" +# specialties: ["Django architecture", "ORM patterns", "REST APIs"] +# disciplines: ["Explicit over implicit", "Reusable apps", "DRY"] +# skillsets: ["Django 4.x", "DRF", "Celery integration"] +# domains: ["Web applications", "REST APIs", "Admin interfaces"] +# perspective: "Leverages Django's batteries-included philosophy effectively" + +# ============================================================================ +# NAMING CONVENTIONS +# ============================================================================ +# +# IDs: Use lowercase with underscores +# - Good: javascript_expert, react_specialist, python_expert +# - Bad: JavaScriptExpert, react-specialist, Python Expert +# +# Names: Use real-sounding names (first and last) +# - Good: "Alex Rivera", "Dr. Sarah Chen", "Marcus Johnson" +# - Bad: "JavaScript Expert", "Python Dev", "The Architect" +# +# Titles: Professional role titles +# - Good: "JavaScript Expert", "React Specialist", "Python Expert" +# - Bad: "JS Guy", "The React Person", "Python Dev" +# +# ============================================================================ +# TIPS FOR GOOD MEMBERS +# ============================================================================ +# +# 1. Be specific: "Modern JavaScript/TypeScript" not just "JavaScript" +# 2. Match your stack: Add specialists for YOUR technologies +# 3. Don't overload: 5-7 technology specialists + core members is plenty +# 4. Unique perspectives: Each member should bring a distinct viewpoint +# 5. Balance depth and breadth: Mix deep specialists with generalists +# 6. Consider project phase: Startups need different members than enterprises +# 7. Team size matters: Larger teams can support more specialists diff --git a/.agents/skills/setup-architect/references/customization-guide.md b/.agents/skills/setup-architect/references/customization-guide.md new file mode 100644 index 00000000..c0bd91cc --- /dev/null +++ b/.agents/skills/setup-architect/references/customization-guide.md @@ -0,0 +1,490 @@ +# Customization Guide + +This document describes how to customize the AI Software Architect framework for your project's specific technology stack, team, and practices. + +## Table of Contents + +1. [Customize Architecture Team Members](#customize-architecture-team-members) +2. [Customize Architectural Principles](#customize-architectural-principles) +3. [Update CLAUDE.md Integration](#update-claudemd-integration) +4. [Configuration Options](#configuration-options) + +--- + +## Customize Architecture Team Members + +The framework uses a roster of specialized architecture reviewers in `.architecture/members.yml`. Customize this roster based on your project's technology stack. + +### Member YAML Structure + +See [../assets/member-template.yml](../assets/member-template.yml) for the complete template. + +Each member includes: +- `id`: Unique identifier (lowercase, underscores) +- `name`: Display name +- `title`: Role/title +- `specialties`: Array of specialty areas +- `disciplines`: Array of discipline focuses +- `skillsets`: Array of specific skills +- `domains`: Array of domain areas +- `perspective`: Description of unique viewpoint + +### Technology Stack-Specific Members + +Add specialists based on your detected technology stack: + +#### JavaScript/TypeScript Projects + +```yaml +- id: javascript_expert + name: "Alex Rivera" + title: "JavaScript Expert" + specialties: + - "Modern JavaScript/TypeScript" + - "Frontend architecture" + - "Build tools" + disciplines: + - "Clean code" + - "Performance" + - "Cross-browser compatibility" + skillsets: + - "ES6+" + - "Async programming" + - "Module systems" + domains: + - "Web applications" + - "Node.js backends" + - "Build pipelines" + perspective: "Focuses on JavaScript best practices and modern language features" +``` + +**Additional specialists to consider**: +- React/Vue/Angular Specialist (if using these frameworks) +- Node.js Expert (for backend projects) +- TypeScript Specialist (for TypeScript-heavy projects) + +#### Python Projects + +```yaml +- id: python_expert + name: "Dr. Sarah Chen" + title: "Python Expert" + specialties: + - "Python best practices" + - "Package architecture" + - "Type hints and mypy" + disciplines: + - "Pythonic code" + - "Testing patterns" + - "Performance optimization" + skillsets: + - "Modern Python (3.10+)" + - "Async/await" + - "Data structures" + domains: + - "Backend services" + - "Data processing" + - "API development" + perspective: "Advocates for clean, Pythonic solutions following PEP guidelines" +``` + +**Additional specialists**: +- Django/Flask/FastAPI Specialist (based on framework) +- Data Engineering Specialist (for data-heavy projects) +- ML/AI Specialist (for ML projects) + +#### Ruby Projects + +```yaml +- id: ruby_expert + name: "Marcus Johnson" + title: "Ruby Expert" + specialties: + - "Ruby idioms" + - "Metaprogramming" + - "Rails patterns" + disciplines: + - "Convention over configuration" + - "DRY principles" + - "Expressive code" + skillsets: + - "Ruby 3.x features" + - "Rails architecture" + - "Gem development" + domains: + - "Web applications" + - "API services" + - "Background jobs" + perspective: "Emphasizes Ruby's elegance and Rails conventions for rapid development" +``` + +**Additional specialists**: +- Rails Architect (for Rails projects) +- Ruby Performance Specialist (for high-traffic applications) + +#### Java Projects + +```yaml +- id: java_expert + name: "Jennifer Park" + title: "Java Expert" + specialties: + - "Enterprise Java" + - "Design patterns" + - "JVM optimization" + disciplines: + - "Object-oriented design" + - "SOLID principles" + - "Clean architecture" + skillsets: + - "Modern Java (17+)" + - "Spring ecosystem" + - "Microservices" + domains: + - "Enterprise applications" + - "Distributed systems" + - "Cloud services" + perspective: "Focuses on maintainable enterprise patterns and modern Java practices" +``` + +**Additional specialists**: +- Spring Boot Specialist +- Microservices Architect + +#### Go Projects + +```yaml +- id: go_expert + name: "Dmitri Ivanov" + title: "Go Expert" + specialties: + - "Idiomatic Go" + - "Concurrency patterns" + - "Service architecture" + disciplines: + - "Simplicity" + - "Composition over inheritance" + - "Clear error handling" + skillsets: + - "Goroutines and channels" + - "Standard library" + - "Performance profiling" + domains: + - "Microservices" + - "Cloud-native applications" + - "CLI tools" + perspective: "Advocates for Go's simplicity philosophy and effective concurrency" +``` + +**Additional specialists**: +- Microservices Architect +- Cloud-Native Specialist + +#### Rust Projects + +```yaml +- id: rust_expert + name: "Katya Volkov" + title: "Rust Expert" + specialties: + - "Memory safety" + - "Zero-cost abstractions" + - "Systems programming" + disciplines: + - "Ownership and borrowing" + - "Type-driven design" + - "Performance optimization" + skillsets: + - "Rust idioms" + - "Async Rust" + - "Unsafe code" + domains: + - "Systems software" + - "Performance-critical code" + - "Embedded systems" + perspective: "Emphasizes Rust's safety guarantees and performance characteristics" +``` + +**Additional specialists**: +- Systems Programmer +- Performance Specialist (Rust-specific) + +### Editing Members + +**To add a member**: +1. Copy the template from `assets/member-template.yml` +2. Fill in all fields appropriately +3. Add to `.architecture/members.yml` in the `members:` array +4. Choose a unique `id` (use lowercase with underscores) + +**To remove a member**: +1. Delete their entry from `.architecture/members.yml` +2. They won't appear in future reviews + +**To modify a member**: +1. Edit their entry in `.architecture/members.yml` +2. Changes apply to all future reviews + +### Core Members (Keep These) + +These core members should remain for all projects: +- **Systems Architect**: Overall architecture coherence +- **Domain Expert**: Business domain representation +- **Security Specialist**: Security analysis +- **Performance Specialist**: Performance and scalability +- **Maintainability Expert**: Code quality and technical debt +- **Pragmatic Enforcer**: YAGNI enforcement (if pragmatic mode enabled) + +**Add technology specialists, don't replace core members.** + +--- + +## Customize Architectural Principles + +The framework's architectural principles in `.architecture/principles.md` should be augmented with framework/technology-specific principles. + +### Framework-Specific Principles + +Add principles specific to your project's frameworks: + +#### React Projects + +**Component Composition**: +- Favor composition over inheritance +- Build reusable, focused components +- Use children props for flexibility + +**State Management**: +- Lift state only when necessary +- Use context for cross-cutting concerns +- Consider state management libraries for complex state + +**Hooks Best Practices**: +- Follow Rules of Hooks +- Custom hooks for reusable logic +- useEffect dependencies matter + +**Props Down, Events Up**: +- Data flows down via props +- Changes flow up via callbacks +- Unidirectional data flow + +#### Rails Projects + +**Convention Over Configuration**: +- Follow Rails conventions +- Don't fight the framework +- Configure only when necessary + +**DRY (Don't Repeat Yourself)**: +- Extract common code +- Use concerns and modules +- Avoid duplication across models/controllers + +**Fat Models, Skinny Controllers**: +- Business logic in models +- Controllers orchestrate only +- Keep views logic-free + +**RESTful Design**: +- Use resourceful routing +- Standard CRUD patterns +- Nested resources where appropriate + +#### Django Projects + +**Explicit Over Implicit**: +- Make behavior clear +- Avoid magic +- Prefer explicit configuration + +**Reusable Apps**: +- Design apps to be reusable +- Clear boundaries and interfaces +- Minimal coupling between apps + +**Use Built-In Features**: +- Leverage Django's admin +- Use ORM effectively +- Don't reinvent wheels + +**Template Inheritance**: +- Base templates for layouts +- Block-based composition +- DRY template code + +### Adding Custom Principles + +**Format**: +```markdown +## [Principle Number]. [Principle Name] + +[Description of the principle and why it matters] + +**In Practice**: +- [Specific application 1] +- [Specific application 2] +- [Specific application 3] + +**Anti-patterns to Avoid**: +- [Anti-pattern 1] +- [Anti-pattern 2] + +**Examples**: +[Code examples showing the principle in action] +``` + +**Where to add**: +- Append to `.architecture/principles.md` +- After the core principles (Livable Code, Clarity over Cleverness, etc.) +- Before any project-specific principles + +--- + +## Update CLAUDE.md Integration + +If your project has a `CLAUDE.md` file for Claude Code integration, append framework usage instructions. + +### Template + +Add this section to your project's `CLAUDE.md`: + +```markdown +## AI Software Architect Framework + +This project uses the AI Software Architect framework for architectural decision tracking and reviews. + +### Available Commands + +- **Create ADR**: "Create ADR for [decision]" +- **Architecture Review**: "Start architecture review for version X.Y.Z" +- **Specialist Review**: "Ask [specialist role] to review [target]" +- **Implementation**: "Implement [feature] as the architects" +- **List Members**: "List architecture members" +- **Status**: "What's our architecture status?" + +### Documentation + +All architecture documentation is in `.architecture/`: +- **ADRs**: `.architecture/decisions/adrs/` +- **Reviews**: `.architecture/reviews/` +- **Principles**: `.architecture/principles.md` +- **Team**: `.architecture/members.yml` +``` + +### Important Notes + +**Do NOT include**: +- Setup instructions (setup is one-time) +- Framework internals +- Implementation details + +**DO include**: +- Usage commands +- Where to find documentation +- How to invoke skills + +### Location + +Append to `CLAUDE.md` in project root (if it exists). If `CLAUDE.md` doesn't exist, you can create it or skip this step. + +--- + +## Configuration Options + +The framework's behavior is controlled via `.architecture/config.yml`. + +### Pragmatic Mode + +```yaml +pragmatic_mode: + enabled: true + intensity: balanced # strict | balanced | lenient + applies_to: + - reviews + - adrs + - implementation + exemptions: + - security + - compliance + - accessibility + - data_loss_prevention +``` + +**Intensity levels**: +- `strict`: Ratio target <1.0 (complexity < necessity) +- `balanced`: Ratio target <1.5 (moderate complexity acceptable) +- `lenient`: Ratio target <2.0 (strategic complexity tolerated) + +**When to enable**: +- **Strict**: MVP, tight timeline, small team +- **Balanced**: Normal development, sustainable pace +- **Lenient**: Platform building, long-term projects + +### Implementation Guidance + +```yaml +implementation: + enabled: true + methodology: TDD # TDD | BDD | DDD | Test-Last | Exploratory + influences: + - "Kent Beck - TDD by Example" + - "Sandi Metz - POODR" + languages: + ruby: + style_guide: "Rubocop" + test_framework: "RSpec" + javascript: + style_guide: "Airbnb" + test_framework: "Jest" +``` + +**Usage**: +When you say "Implement [feature] as the architects", the framework applies this methodology automatically. + +### Review Configuration + +```yaml +reviews: + frequency: + major_versions: required + features: optional + regular: quarterly + auto_track_actions: true + require_success_metrics: true +``` + +--- + +## Customization Checklist + +After installation, customize: + +- [ ] Add technology-specific members to `.architecture/members.yml` +- [ ] Add framework-specific principles to `.architecture/principles.md` +- [ ] Update `.architecture/config.yml` with team preferences +- [ ] Append framework usage to `CLAUDE.md` (if exists) +- [ ] Review and adjust pragmatic mode settings +- [ ] Configure implementation methodology (if using) +- [ ] Run initial system analysis +- [ ] Verify setup with "What's our architecture status?" + +--- + +## Best Practices + +**Do**: +- Customize based on actual project needs +- Add specialists for primary technologies +- Keep core members (Systems, Domain, Security, etc.) +- Tailor principles to your stack +- Enable pragmatic mode appropriately + +**Don't**: +- Add every possible specialist (overwhelms reviews) +- Remove core members (they provide essential perspectives) +- Copy principles verbatim without customization +- Enable strict pragmatic mode for established projects +- Skip initial analysis (provides valuable baseline) + +For the initial analysis template, see [../assets/initial-analysis-template.md](../assets/initial-analysis-template.md). diff --git a/.agents/skills/setup-architect/references/installation-procedures.md b/.agents/skills/setup-architect/references/installation-procedures.md new file mode 100644 index 00000000..594b5f30 --- /dev/null +++ b/.agents/skills/setup-architect/references/installation-procedures.md @@ -0,0 +1,403 @@ +# Installation Procedures - Detailed Guide + +This document provides detailed step-by-step procedures for installing the AI Software Architect framework in a project. + +## Table of Contents + +1. [Prerequisites Verification](#prerequisites-verification) +2. [Framework Installation](#framework-installation) +3. [Agent Documentation Setup](#agent-documentation-setup) +4. [Cleanup Procedures](#cleanup-procedures) +5. [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites Verification + +Before installing, verify the environment is ready. + +### Check Framework is Cloned + +The framework must be cloned into `.architecture/.architecture/`: + +```bash +if [ ! -d ".architecture/.architecture" ]; then + echo "❌ Framework not found. Please clone first:" + echo " git clone https://github.com/codenamev/ai-software-architect .architecture/.architecture" + exit 1 +fi + +echo "✅ Framework found at .architecture/.architecture" +``` + +### Confirm Project Root + +Verify we're in the project root directory: + +```bash +# Look for common project markers +if [ -f "package.json" ] || [ -f "Gemfile" ] || [ -f "requirements.txt" ] || [ -f "go.mod" ] || [ -f "Cargo.toml" ]; then + echo "✅ In project root" +else + echo "⚠️ No project markers found. Are you in the project root?" + # Continue but warn user +fi +``` + +--- + +## Framework Installation + +### Step 1: Copy Framework Files + +Copy the framework from the cloned location to `.architecture/`: + +```bash +# Copy framework files (they're in the .architecture subfolder of the cloned repo) +cp -r .architecture/.architecture/.architecture/* .architecture/ + +# Verify copy succeeded +if [ $? -eq 0 ]; then + echo "✅ Framework files copied" +else + echo "❌ Copy failed" + exit 1 +fi +``` + +### Step 2: Remove Clone Directory + +Clean up the temporary clone directory: + +```bash +# Remove the clone directory (no longer needed) +rm -rf .architecture/.architecture + +if [ ! -d ".architecture/.architecture" ]; then + echo "✅ Clone directory removed" +fi +``` + +### Step 3: Create Directory Structure + +Create all required directories: + +```bash +# Create coding assistant directories +mkdir -p .coding-assistants/claude +mkdir -p .coding-assistants/cursor +mkdir -p .coding-assistants/codex + +# Create architecture directories +mkdir -p .architecture/decisions/adrs +mkdir -p .architecture/reviews +mkdir -p .architecture/recalibration +mkdir -p .architecture/comparisons +mkdir -p .architecture/agent_docs + +echo "✅ Directory structure created" +``` + +### Step 4: Initialize Configuration + +Copy the default configuration file: + +```bash +# Copy config template if exists +if [ -f ".architecture/templates/config.yml" ]; then + cp .architecture/templates/config.yml .architecture/config.yml + echo "✅ Configuration initialized" +else + echo "⚠️ No config template found" +fi +``` + +**Verify installation**: +```bash +# Check key directories exist +test -d .architecture/decisions/adrs && \ +test -d .architecture/reviews && \ +test -f .architecture/members.yml && \ +test -f .architecture/principles.md && \ +echo "✅ Installation verified" || echo "❌ Installation incomplete" +``` + +--- + +## Agent Documentation Setup + +Following ADR-006 (Progressive Disclosure), create agent-specific documentation. + +### Copy Existing Agent Docs (If Available) + +If the framework includes agent documentation, copy it as templates: + +```bash +if [ -d ".architecture/agent_docs" ]; then + # Backup existing files as templates + if [ -f ".architecture/agent_docs/workflows.md" ]; then + cp .architecture/agent_docs/workflows.md .architecture/agent_docs/workflows.md.template + fi + + if [ -f ".architecture/agent_docs/reference.md" ]; then + cp .architecture/agent_docs/reference.md .architecture/agent_docs/reference.md.template + fi + + if [ -f ".architecture/agent_docs/README.md" ]; then + cp .architecture/agent_docs/README.md .architecture/agent_docs/README.md.template + fi + + echo "✅ Agent docs backed up as templates" +fi +``` + +### Create Agent Documentation Files + +Create three core documentation files: + +**1. workflows.md** - Procedural documentation: +- Setup procedures (Claude Skills, Direct Clone, MCP) +- Architecture review process +- ADR creation workflow +- Implementation with methodology +- Step-by-step instructions for common tasks + +**2. reference.md** - Reference documentation: +- Pragmatic mode details and intensity levels +- Recalibration process +- Advanced configuration options +- Troubleshooting guide +- Configuration examples + +**3. README.md** - Navigation guide: +- Progressive disclosure explanation +- Quick navigation table (task → section) +- How to find information +- When to read which document + +**Content Guidelines**: +- AGENTS.md: ~400 lines, always-relevant overview +- agent_docs/: Task-specific details loaded as needed +- Keep workflows procedural and actionable +- Reference should be comprehensive but organized +- README should help users navigate effectively + +--- + +## Cleanup Procedures + +Remove framework development files that shouldn't be in user projects. + +### Remove Documentation Files + +```bash +# Remove framework documentation (users don't need these) +rm -f .architecture/README.md +rm -f .architecture/USAGE*.md +rm -f .architecture/INSTALL.md + +echo "✅ Framework docs removed" +``` + +### Remove Framework Git Repository + +**⚠️ CRITICAL SAFEGUARDS - READ CAREFULLY** + +Removing `.git` directory is destructive. Follow these safeguards: + +#### Safeguard 1: Verify Project Root + +```bash +# Check we're in project root (NOT in .architecture/) +if [ ! -f "package.json" ] && [ ! -f ".git/config" ] && [ ! -f "Gemfile" ]; then + echo "❌ ERROR: Not in project root. Stopping." + echo " Current directory: $(pwd)" + exit 1 +fi + +echo "✅ Verified in project root" +``` + +#### Safeguard 2: Verify Target Exists + +```bash +# Check .architecture/.git exists before attempting removal +if [ ! -d ".architecture/.git" ]; then + echo "✅ No .git directory to remove" + exit 0 +fi + +echo "⚠️ Found .architecture/.git - proceeding with verification" +``` + +#### Safeguard 3: Verify It's the Template Repo + +```bash +# Verify .git/config contains template repository URL +if ! grep -q "ai-software-architect" .architecture/.git/config 2>/dev/null; then + echo "❌ ERROR: .architecture/.git doesn't appear to be template repo" + echo " Found config:" + cat .architecture/.git/config 2>/dev/null || echo " (could not read config)" + echo "" + echo "⛔ STOPPING - User confirmation required" + exit 1 +fi + +echo "✅ Verified template repository" +``` + +#### Safeguard 4: Use Absolute Path + +```bash +# Get absolute path (never use relative paths with rm -rf) +ABS_PATH="$(pwd)/.architecture/.git" + +echo "Removing: $ABS_PATH" + +# Verify path is what we expect +if [[ "$ABS_PATH" != *"/.architecture/.git" ]]; then + echo "❌ ERROR: Path doesn't match expected pattern" + echo " Path: $ABS_PATH" + exit 1 +fi + +echo "✅ Path verified" +``` + +#### Safeguard 5: Execute Removal + +```bash +# Remove with absolute path (no wildcards!) +rm -rf "$ABS_PATH" + +# Verify removal +if [ ! -d ".architecture/.git" ]; then + echo "✅ Template .git removed successfully" +else + echo "⚠️ .git directory still exists" +fi +``` + +**Complete Safe Removal Script**: + +```bash +#!/bin/bash +# Safe removal of template repository .git directory + +set -e # Exit on any error + +echo "=== Safe .git Removal ===" + +# 1. Verify project root +if [ ! -f "package.json" ] && [ ! -f ".git/config" ] && [ ! -f "Gemfile" ]; then + echo "❌ Not in project root" + exit 1 +fi + +# 2. Check target exists +if [ ! -d ".architecture/.git" ]; then + echo "✅ No .git to remove" + exit 0 +fi + +# 3. Verify template repo +if ! grep -q "ai-software-architect" .architecture/.git/config 2>/dev/null; then + echo "❌ Not template repo - STOPPING" + exit 1 +fi + +# 4. Get absolute path +ABS_PATH="$(pwd)/.architecture/.git" + +# 5. Verify path pattern +if [[ "$ABS_PATH" != *"/.architecture/.git" ]]; then + echo "❌ Unexpected path - STOPPING" + exit 1 +fi + +# 6. Execute removal +echo "Removing: $ABS_PATH" +rm -rf "$ABS_PATH" + +# 7. Verify success +if [ ! -d ".architecture/.git" ]; then + echo "✅ Successfully removed" +else + echo "❌ Removal failed" + exit 1 +fi +``` + +--- + +## Troubleshooting + +### Common Issues + +**Issue**: "Framework not found at .architecture/.architecture" +- **Cause**: Framework not cloned +- **Solution**: `git clone https://github.com/codenamev/ai-software-architect .architecture/.architecture` + +**Issue**: "Permission denied" errors during copy +- **Cause**: Insufficient file permissions +- **Solution**: `chmod -R u+rw .architecture/` + +**Issue**: "Directory already exists" during mkdir +- **Cause**: Framework already partially installed +- **Solution**: Check if framework is already set up: `ls -la .architecture/` + +**Issue**: ".git removal verification failed" +- **Cause**: Safety check detected unexpected repository +- **Solution**: Manually verify `.architecture/.git/config` contains template repo URL +- **Never**: Override safety checks without understanding why they failed + +**Issue**: "No project markers found" +- **Cause**: May not be in project root +- **Solution**: Verify you're in the correct directory, proceed with caution + +### Verification Commands + +**Check installation completeness**: +```bash +# Required directories +test -d .architecture/decisions/adrs && echo "✅ ADRs directory" || echo "❌ Missing ADRs" +test -d .architecture/reviews && echo "✅ Reviews directory" || echo "❌ Missing reviews" + +# Required files +test -f .architecture/members.yml && echo "✅ Members file" || echo "❌ Missing members" +test -f .architecture/principles.md && echo "✅ Principles file" || echo "❌ Missing principles" +test -f .architecture/config.yml && echo "✅ Config file" || echo "❌ Missing config" +``` + +**Check for leftover framework files**: +```bash +# These should NOT exist after cleanup +test -f .architecture/README.md && echo "⚠️ Framework README still present" +test -d .architecture/.git && echo "⚠️ Template .git still present" +test -d .architecture/.architecture && echo "⚠️ Clone directory still present" +``` + +### Recovery + +**If installation fails mid-process**: +1. Remove partial installation: `rm -rf .architecture/` (if nothing important there yet) +2. Re-clone framework: `git clone https://github.com/codenamev/ai-software-architect .architecture/.architecture` +3. Start over from Step 1 + +**If you accidentally removed the wrong .git**: +- If it was your project's .git: **Restore from backup immediately** +- If you don't have a backup: Recovery may not be possible +- This is why the safeguards are critical + +--- + +## Post-Installation + +After installation is complete: + +1. **Verify setup**: Run `"What's our architecture status?"` +2. **Review customizations**: Check `.architecture/members.yml` and `.architecture/principles.md` +3. **Run initial analysis**: The setup process creates an initial system analysis +4. **Create first ADR**: Document an early architectural decision + +For customization procedures, see [customization-guide.md](./customization-guide.md). diff --git a/.agents/skills/specialist-review/SKILL.md b/.agents/skills/specialist-review/SKILL.md new file mode 100644 index 00000000..26011fc3 --- /dev/null +++ b/.agents/skills/specialist-review/SKILL.md @@ -0,0 +1,199 @@ +--- +name: specialist-review +description: Conducts a focused review from ONE specific specialist's perspective (e.g., Security Specialist, Performance Expert). Use when the user requests "Ask [specialist role] to review [target]", "Get [specialist]'s opinion on [topic]", "Have [role] review [code/component]", or when they want deep expertise in ONE specific domain. Do NOT use for comprehensive multi-perspective reviews (use architecture-review instead) or for listing available specialists (use list-members instead). +allowed-tools: Read,Write,Glob,Grep +--- + +# Specialist Review + +Conducts focused reviews from a specific specialist's perspective. + +## Overview + +This skill performs a deep-dive review from one specialist's expertise: +1. Parses which specialist and what target to review +2. Loads or creates the specialist in the team +3. Analyzes the target from that specialist's unique lens +4. Conducts expert-level review with specific findings +5. Generates detailed review document +6. Reports key findings and recommendations + +**Specialist guidance**: [references/specialist-perspectives.md](references/specialist-perspectives.md) +**Review template**: [assets/specialist-review-template.md](assets/specialist-review-template.md) + +## High-Level Workflow + +### 1. Parse Request + +Extract from user request: +- **Specialist role**: Which expert? (e.g., "Security Specialist", "Performance Expert") +- **Target**: What to review? (e.g., "API authentication", "database queries") + +**Input validation**: Apply sanitization from `_patterns.md`: +- Specialist role: Alphanumeric + spaces/hyphens only, convert to kebab-case for filename +- Target: Remove dangerous characters, convert to kebab-case +- Combined filename length: max 100 characters + +**Examples**: +- "Security Specialist" + "API authentication" → `security-specialist-api-authentication.md` +- "Ruby Expert" + "ActiveRecord models" → `ruby-expert-activerecord-models.md` + +### 2. Load or Create Specialist + +Check `.architecture/members.yml` for the requested specialist. + +**If exists**: Load their profile (specialties, disciplines, domains, perspective) + +**If doesn't exist**: Create new member and add to `members.yml`: +```yaml +- id: [specialist_id] + name: "[Person Name]" + title: "[Specialist Title]" + specialties: ["[Specialty 1]", "[Specialty 2]", "[Specialty 3]"] + disciplines: ["[Discipline 1]", "[Discipline 2]"] + skillsets: ["[Skill 1]", "[Skill 2]"] + domains: ["[Domain 1]", "[Domain 2]"] + perspective: "[Their unique viewpoint]" +``` + +Inform user: "I've added [Name] ([Title]) to your architecture team." + +**Specialist guidance**: See [references/specialist-perspectives.md § Creating New Specialists](references/specialist-perspectives.md#creating-new-specialists) + +### 3. Analyze Target + +Use available tools to examine the target: +- `Glob` to find relevant files +- `Grep` to search for patterns +- `Read` to examine code, configs, documentation + +**Understand**: +- Current implementation +- Dependencies and context +- Related ADRs or documentation +- Patterns being used + +### 4. Conduct Expert Review + +Adopt the specialist's persona and expertise. Apply their unique lens. + +**Review from specialist's perspective**: +- Focus on their domain of expertise (security, performance, maintainability, etc.) +- Provide expert-level insights, not surface-level comments +- Reference specific files, line numbers, and code +- Explain impact and provide actionable fixes + +**Review structure** (for each specialist): +- Specialist perspective and focus +- Executive summary with assessment +- Current implementation description +- Strengths identified +- Concerns with severity and specific fixes +- Recommendations (immediate, short-term, long-term) +- Best practices and industry standards +- Code examples showing issues and improvements +- Risks if not addressed +- Success metrics + +**Detailed guidance by specialist**: [references/specialist-perspectives.md § Core Specialists](references/specialist-perspectives.md#core-specialists) + +**Review template**: Load and fill [assets/specialist-review-template.md](assets/specialist-review-template.md) + +### 5. Create Review Document + +Load the template: +```bash +cat .claude/skills/specialist-review/assets/specialist-review-template.md +``` + +Fill in all sections with detailed, specific findings. + +**Save to**: `.architecture/reviews/[specialist-role]-[target].md` + +**Format**: `[role-kebab-case]-[target-kebab-case].md` + +### 6. Report to User + +Provide concise summary: + +``` +[Specialist Title] Review Complete: [Target] + +Reviewer: [Specialist Name] +Location: .architecture/reviews/[filename].md +Assessment: [Overall assessment] + +Key Findings: +1. [Most important finding] +2. [Second finding] +3. [Third finding] + +Priority Actions: +1. [Critical action 1] +2. [Critical action 2] + +Critical Issues: [Count] +High Priority: [Count] +Total Recommendations: [Count] + +Next Steps: +- Address critical issues immediately +- Review detailed findings in document +- [Specific next action based on findings] +``` + +## Specialist Quick Reference + +**Core Specialists** (see [references/specialist-perspectives.md](references/specialist-perspectives.md)): +- **Security Specialist**: Authentication, authorization, vulnerabilities, OWASP +- **Performance Specialist**: Query optimization, caching, bottlenecks, scalability +- **Domain Expert**: Business logic, domain models, ubiquitous language +- **Maintainability Expert**: Code quality, technical debt, testability +- **Systems Architect**: Architecture patterns, component interaction, coherence +- **AI Engineer**: LLM integration, agent orchestration, evaluation + +**Technology Specialists**: +- **JavaScript/Python/Ruby/Go/Rust Expert**: Language-specific best practices +- **Framework Specialists**: React, Rails, Django, Spring, etc. + +**Creating new specialists**: Automatically added to team when requested + +## Related Skills + +**Before Specialist Review**: +- `list-members` - See available specialists +- `architecture-status` - Check if area previously reviewed + +**After Specialist Review**: +- `create-adr` - Document decisions from findings +- `architecture-review` - Include in comprehensive review +- Request another specialist for different domain perspective + +**Workflow Examples**: +1. Security review → Finds auth issue → Create ADR → Performance review +2. Ruby Expert review → Rails-specific guidance → Implement → Follow-up review +3. Full architecture review → Deep-dive with specialists on concerns + +## Quality Guidelines + +**Excellent specialist reviews**: +- Stay laser-focused within domain +- Provide expert-level, not generic, insights +- Reference exact files and line numbers +- Include code examples (current vs recommended) +- Explain "why", not just "what" +- Consider context and constraints +- Provide actionable, implementable advice +- Estimate effort for each recommendation + +**Avoid**: +- Straying outside specialist's domain +- Vague or surface-level comments +- Missing specific locations +- Recommendations without implementation guidance + +## Documentation + +- **Specialist guidance**: [references/specialist-perspectives.md](references/specialist-perspectives.md) +- **Review template**: [assets/specialist-review-template.md](assets/specialist-review-template.md) +- **Common patterns**: [../_patterns.md](../_patterns.md) diff --git a/.agents/skills/specialist-review/assets/specialist-review-template.md b/.agents/skills/specialist-review/assets/specialist-review-template.md new file mode 100644 index 00000000..9e197cd1 --- /dev/null +++ b/.agents/skills/specialist-review/assets/specialist-review-template.md @@ -0,0 +1,296 @@ +# [Specialist Title] Review: [Target] + +**Reviewer**: [Specialist Name], [Specialist Title] +**Target**: [What's being reviewed - be specific] +**Date**: [YYYY-MM-DD] +**Review Type**: Specialist Review + +--- + +## Specialist Perspective + +**Focus**: [What this specialist looks for based on their expertise] + +[Brief explanation of this specialist's unique lens and what they prioritize in reviews] + +--- + +## Executive Summary + +[Write 2-3 sentences summarizing the review findings] + +**Overall Assessment**: Excellent | Good | Adequate | Needs Improvement | Critical Issues + +**Key Findings**: +- [Most significant finding] +- [Second most significant finding] +- [Third most significant finding] + +**Critical Actions Required**: [Number of critical items] + +--- + +## Current Implementation + +[Describe what was reviewed with specific file references and context] + +**Scope Reviewed**: +- [Component/feature 1] +- [Component/feature 2] +- [Component/feature 3] + +**Key Components**: +- `[file.ext:line]`: [Brief description of component] +- `[file.ext:line]`: [Brief description] +- `[file.ext:line]`: [Brief description] + +**Pattern/Approach Used**: [Describe the pattern or approach currently implemented] + +--- + +## Assessment + +### Strengths + +[Identify what's working well from this specialist's perspective] + +1. **[Strength Title]**: [Description of what's good and why it matters from this specialist's viewpoint] + +2. **[Strength Title]**: [Description] + +3. **[Strength Title]**: [Description] + +[Aim for 3-5 strengths - recognize good practices] + +### Concerns + +[Identify issues, weaknesses, and areas needing improvement] + +1. **[Concern Title]** (Severity: Critical | High | Medium | Low) + - **Issue**: [Clear description of what's wrong or could be improved] + - **Location**: `[file.ext:line-range]` + - **Impact**: [What problems this causes - performance, security, maintainability, etc.] + - **Fix**: [Specific, actionable recommendation] + - **Effort**: Small | Medium | Large + +2. **[Concern Title]** (Severity: Critical | High | Medium | Low) + - **Issue**: [Description] + - **Location**: `[file.ext:line]` + - **Impact**: [Impact] + - **Fix**: [Recommendation] + - **Effort**: Small | Medium | Large + +3. **[Concern Title]** (Severity: Critical | High | Medium | Low) + - [Details] + +[Continue with additional concerns - prioritize by severity] + +### Observations + +[Neutral observations that aren't necessarily problems but worth noting] + +- **[Observation]**: [Description and context] +- **[Observation]**: [Description] +- **[Observation]**: [Description] + +--- + +## Recommendations + +### Immediate (0-2 Weeks) + +[Critical and high-priority items that should be addressed right away] + +1. **[Recommendation Title]** + - **What**: [Specific action to take] + - **Why**: [Reason and benefit] + - **How**: [Implementation approach] + - **Effort**: Small | Medium | Large + - **Priority**: Critical | High + +2. **[Recommendation]** + - [Details] + +### Short-term (2-8 Weeks) + +[Important improvements to address in the near term] + +1. **[Recommendation Title]** + - **What**: [Action] + - **Why**: [Reason] + - **How**: [Approach] + - **Effort**: Small | Medium | Large + - **Priority**: Medium + +2. **[Recommendation]** + - [Details] + +### Long-term (2-6 Months) + +[Strategic improvements for future consideration] + +1. **[Recommendation Title]** + - **What**: [Action] + - **Why**: [Strategic value] + - **How**: [High-level approach] + - **Effort**: Medium | Large + - **Priority**: Low | Nice-to-Have + +2. **[Recommendation]** + - [Details] + +--- + +## Best Practices + +[Industry best practices relevant to this specialist's domain] + +1. **[Practice Title]**: [Description of best practice and how it applies to this codebase] + +2. **[Practice Title]**: [Description] + +3. **[Practice Title]**: [Description] + +**Industry Standards**: [Reference relevant standards, guidelines, or frameworks] +- [Standard 1]: [How it applies] +- [Standard 2]: [How it applies] + +--- + +## Code Examples + +[Provide concrete code examples showing current issues and recommended improvements] + +### Example 1: [Issue Title] + +**Current Implementation (Problematic)**: +```[language] +[Code snippet showing the concern] +``` + +**Issues**: +- [Issue 1] +- [Issue 2] +- [Issue 3] + +**Recommended Improvement**: +```[language] +[Code snippet showing the improved approach] +``` + +**Benefits**: +- [Benefit 1] +- [Benefit 2] +- [Benefit 3] + +### Example 2: [Issue Title] + +[Repeat structure for additional examples] + +--- + +## Risks + +**If Recommendations Not Addressed**: + +1. **[Risk Title]** (Likelihood: High | Medium | Low, Impact: High | Medium | Low) + - **Description**: [What could go wrong] + - **Timeframe**: [When this might occur] + - **Impact**: [Consequences] + - **Mitigation**: [How to reduce or eliminate risk] + +2. **[Risk Title]** (Likelihood, Impact) + - **Description**: [Risk description] + - **Timeframe**: [When] + - **Impact**: [Consequences] + - **Mitigation**: [How to address] + +3. **[Risk Title]** (Likelihood, Impact) + - [Details] + +--- + +## Success Metrics + +[Define how to measure improvement after implementing recommendations] + +1. **[Metric Name]** + - **Current**: [Baseline value] + - **Target**: [Desired value] + - **Timeline**: [When to achieve] + - **How to Measure**: [Measurement method] + +2. **[Metric Name]**: Current → Target (Timeline) + +3. **[Metric Name]**: Current → Target (Timeline) + +[Examples: +- Test Coverage: 40% → 80% (6 weeks) +- Response Time: 2s → 300ms (4 weeks) +- Security Score: 6/10 → 9/10 (8 weeks) +] + +--- + +## Follow-up + +**Re-Review Recommended**: [When to conduct follow-up review] + +**Success Criteria for Closure**: +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [Criterion 3] + +**Next Steps**: +1. [Immediate next step] +2. [Second next step] +3. [Third next step] + +**Related Reviews**: +- [Link to related reviews if any] + +**Related ADRs**: +- [Link to relevant ADRs if decisions are needed] + +--- + +## Appendix + +### Review Scope + +**What Was Reviewed**: +- [Scope item 1] +- [Scope item 2] +- [Scope item 3] + +**What Was Not Reviewed**: +- [Out of scope item 1] +- [Out of scope item 2] + +### Methodology + +This review was conducted by analyzing: +- Code structure and implementation +- [Relevant documentation] +- [Test coverage] +- [Performance metrics] +- [Security scan results] +- [Other relevant inputs] + +### References + +**Documentation Reviewed**: +- [Document 1] +- [Document 2] + +**Standards Referenced**: +- [Standard 1] +- [Standard 2] + +**Tools Used**: +- [Tool 1] +- [Tool 2] + +--- + +**Review Complete** diff --git a/.agents/skills/specialist-review/references/specialist-perspectives.md b/.agents/skills/specialist-review/references/specialist-perspectives.md new file mode 100644 index 00000000..79754b29 --- /dev/null +++ b/.agents/skills/specialist-review/references/specialist-perspectives.md @@ -0,0 +1,511 @@ +# Specialist Perspectives - Review Guidance + +This document provides detailed guidance for conducting reviews from each specialist's unique perspective. + +## Table of Contents + +1. [General Review Approach](#general-review-approach) +2. [Core Specialists](#core-specialists) +3. [Technology Specialists](#technology-specialists) +4. [Creating New Specialists](#creating-new-specialists) + +--- + +## General Review Approach + +All specialist reviews should follow these principles: + +### Stay Focused + +**Do**: +- Laser-focus on your domain of expertise +- Deep-dive within your specialty +- Provide expert-level insights + +**Don't**: +- Stray into other specialists' domains +- Provide surface-level general comments +- Dilute expertise by being too broad + +### Be Specific + +**Always include**: +- Exact file paths and line numbers +- Concrete code examples +- Specific recommendations + +**Example**: +- ❌ "The authentication is insecure" +- ✅ "The authentication in `src/auth/login.ts:45` uses MD5 hashing, which is cryptographically broken. Migrate to bcrypt with salt rounds ≥12." + +### Provide Context + +**Explain**: +- Why this matters (impact) +- Industry standards or best practices +- Trade-offs of recommendations +- Effort required + +### Be Actionable + +**Every concern should include**: +- What's wrong +- Where it is (file:line) +- Impact if not fixed +- Specific fix with implementation approach +- Effort estimate + +--- + +## Core Specialists + +These specialists are part of the framework's core team. + +### Security Specialist + +**Role**: Identifies security vulnerabilities, threats, and risks. + +**Focus Areas**: +- **Authentication**: Login mechanisms, password handling, session management +- **Authorization**: Access control, permissions, role-based access +- **Input Validation**: XSS, SQL injection, command injection prevention +- **Data Protection**: Encryption at rest/transit, PII handling, secrets management +- **OWASP Top 10**: Cross-check against current OWASP vulnerabilities +- **Compliance**: GDPR, HIPAA, PCI-DSS requirements +- **API Security**: Rate limiting, CORS, API keys, OAuth flows +- **Dependencies**: Known CVEs in libraries + +**Review Checklist**: +- [ ] Authentication mechanism reviewed +- [ ] Authorization boundaries checked +- [ ] All user inputs validated +- [ ] Sensitive data encrypted +- [ ] Secrets not hardcoded +- [ ] HTTPS enforced +- [ ] CORS configured correctly +- [ ] Rate limiting implemented +- [ ] Dependencies scanned for CVEs + +**Severity Levels**: +- **Critical**: Exploitable vulnerability, immediate data breach risk +- **High**: Security weakness requiring prompt attention +- **Medium**: Security concern to address in near term +- **Low**: Security improvement opportunity + +**Example Concerns**: +``` +1. **SQL Injection Vulnerability** (Severity: Critical) + - **Issue**: User input directly interpolated into SQL query + - **Location**: `src/db/users.js:23` + - **Impact**: Attacker can read/modify/delete any database record + - **Fix**: Use parameterized queries with prepared statements + - **Code**: + ```javascript + // Current (VULNERABLE) + const query = `SELECT * FROM users WHERE email = '${email}'`; + + // Recommended (SAFE) + const query = 'SELECT * FROM users WHERE email = ?'; + db.query(query, [email]); + ``` +``` + +**Best Practices to Reference**: +- OWASP Secure Coding Practices +- CWE (Common Weakness Enumeration) +- NIST guidelines +- Industry security standards + +--- + +### Performance Specialist + +**Role**: Optimizes system performance, scalability, and resource utilization. + +**Focus Areas**: +- **Query Optimization**: Database query efficiency, indexes, N+1 queries +- **Caching**: Cache strategies, TTL, invalidation, cache hits +- **Resource Utilization**: CPU, memory, disk I/O, network +- **Bottlenecks**: Profiling, hotspots, performance critical paths +- **Load Handling**: Concurrency, async operations, connection pooling +- **Scalability**: Horizontal/vertical scaling, load balancing +- **Frontend Performance**: Bundle size, lazy loading, rendering + +**Review Checklist**: +- [ ] Database queries optimized +- [ ] Proper indexing in place +- [ ] N+1 query problems identified +- [ ] Caching opportunities evaluated +- [ ] Async operations for I/O +- [ ] Resource usage profiled +- [ ] Scalability concerns addressed +- [ ] Frontend bundle size reasonable + +**Metrics to Measure**: +- Response time (p50, p95, p99) +- Throughput (requests/second) +- Database query time +- Cache hit rate +- Memory usage +- CPU utilization + +**Example Concerns**: +``` +1. **N+1 Query Problem** (Severity: High) + - **Issue**: Loading users in loop triggers 1000+ database queries + - **Location**: `src/controllers/orders.js:45-52` + - **Impact**: 15-second page load time, database overload + - **Fix**: Use eager loading to fetch all users in single query + - **Code**: + ```javascript + // Current (INEFFICIENT - N+1 queries) + const orders = await Order.findAll(); + for (const order of orders) { + order.user = await User.findById(order.userId); // N queries! + } + + // Recommended (EFFICIENT - 1 query) + const orders = await Order.findAll({ + include: [{ model: User }] + }); + ``` + - **Expected Improvement**: 15s → 0.3s (50x faster) +``` + +--- + +### Domain Expert + +**Role**: Ensures architecture accurately represents business domain and concepts. + +**Focus Areas**: +- **Domain Models**: Entities, value objects, aggregates +- **Business Logic**: Rules, workflows, calculations +- **Ubiquitous Language**: Consistent terminology +- **Bounded Contexts**: Clear domain boundaries +- **Business Rules**: Validation, invariants, constraints +- **Semantic Accuracy**: Code reflects domain accurately + +**Review Checklist**: +- [ ] Domain concepts clearly modeled +- [ ] Business logic in domain layer (not scattered) +- [ ] Ubiquitous language used consistently +- [ ] Bounded contexts well-defined +- [ ] Business rules enforced +- [ ] Domain invariants protected + +**Example Concerns**: +``` +1. **Anemic Domain Model** (Severity: Medium) + - **Issue**: `Order` class is data container with no business logic + - **Location**: `src/models/Order.js` + - **Impact**: Business logic scattered in controllers, hard to maintain + - **Fix**: Move order calculation and validation into Order class + - **Code**: + ```javascript + // Current (ANEMIC) + class Order { + constructor(items) { + this.items = items; + } + } + // Business logic in controller + const total = order.items.reduce((sum, item) => sum + item.price, 0); + + // Recommended (RICH DOMAIN MODEL) + class Order { + constructor(items) { + this.items = items; + } + + calculateTotal() { + return this.items.reduce((sum, item) => sum + item.price, 0); + } + + applyDiscount(discount) { + if (discount < 0 || discount > 1) { + throw new InvalidDiscountError(); + } + // Domain logic stays with domain object + } + } + ``` +``` + +--- + +### Maintainability Expert + +**Role**: Ensures code is maintainable, evolvable, and understandable. + +**Focus Areas**: +- **Code Quality**: Readability, clarity, simplicity +- **Technical Debt**: Code smells, anti-patterns, cruft +- **Documentation**: Comments, READMEs, inline docs +- **Testability**: Unit tests, test coverage, mocking +- **Complexity**: Cyclomatic complexity, coupling, cohesion +- **Refactoring**: Opportunities to improve structure + +**Review Checklist**: +- [ ] Code is readable and clear +- [ ] Functions/methods are focused +- [ ] Complexity is reasonable +- [ ] Technical debt identified +- [ ] Tests exist and are meaningful +- [ ] Documentation is adequate +- [ ] Code smells flagged + +**Code Smells to Watch For**: +- Long methods (>20-30 lines) +- Large classes (>300 lines) +- Deep nesting (>3 levels) +- Duplicate code +- Magic numbers +- God objects +- Feature envy + +**Example Concerns**: +``` +1. **God Class Anti-pattern** (Severity: Medium) + - **Issue**: `UserService` has 45 methods handling auth, profile, notifications, billing + - **Location**: `src/services/UserService.js` (1,200 lines) + - **Impact**: Hard to understand, test, modify; violates single responsibility + - **Fix**: Split into focused services + - **Recommended Structure**: + ``` + src/services/ + ├── AuthenticationService.js (login, logout, sessions) + ├── UserProfileService.js (profile CRUD) + ├── NotificationService.js (user notifications) + └── BillingService.js (user billing) + ``` + - **Benefit**: Each service <300 lines, single responsibility, easier testing +``` + +--- + +### Systems Architect + +**Role**: Evaluates overall system coherence, patterns, and architectural decisions. + +**Focus Areas**: +- **Architecture Patterns**: MVC, CQRS, Event Sourcing, Microservices +- **Component Interaction**: How pieces fit together +- **Separation of Concerns**: Proper layering and boundaries +- **Scalability**: System design for growth +- **Integration**: External system connections +- **Consistency**: Architectural principles applied consistently + +**Review Checklist**: +- [ ] Architecture pattern appropriately applied +- [ ] Components properly separated +- [ ] Dependencies flow correctly +- [ ] Scalability considered +- [ ] Integration points well-designed +- [ ] Architectural principles followed + +--- + +### AI Engineer + +**Role**: Reviews AI/ML integration, LLM applications, and agent systems. + +**Focus Areas**: +- **LLM Integration**: Prompt design, context management, token efficiency +- **Agent Orchestration**: Multi-agent coordination, task decomposition +- **Evaluation**: Metrics, benchmarks, quality assessment +- **Observability**: Logging, tracing, debugging AI behavior +- **RAG Systems**: Retrieval strategies, context relevance +- **Prompt Engineering**: Template design, chain-of-thought, few-shot + +**Review Checklist**: +- [ ] Prompts are well-designed and tested +- [ ] Context windows managed efficiently +- [ ] Evaluation metrics defined +- [ ] Observability implemented +- [ ] Error handling for AI failures +- [ ] Cost management (token usage) + +--- + +## Technology Specialists + +Add these when reviewing technology-specific code. + +### JavaScript Expert + +**Focus**: Modern JavaScript/TypeScript, async patterns, ES6+, Node.js + +**Key Areas**: +- Promises, async/await usage +- Error handling patterns +- Module system (ESM/CommonJS) +- TypeScript type safety +- Memory leaks in closures +- Event loop understanding + +**Common Issues**: +- Unhandled promise rejections +- Callback hell +- Blocking the event loop +- Type assertions bypassing safety +- Improper this binding + +--- + +### Python Expert + +**Focus**: Pythonic code, PEP compliance, async patterns + +**Key Areas**: +- PEP 8 style compliance +- Type hints (mypy) +- Generator/iterator usage +- Context managers +- Async/await patterns +- Package structure + +**Common Issues**: +- Non-Pythonic code +- Mutable default arguments +- Global state misuse +- Blocking I/O in async code +- Import cycles + +--- + +### Ruby Expert + +**Focus**: Ruby idioms, Rails conventions, metaprogramming + +**Key Areas**: +- Ruby idioms and patterns +- Rails conventions +- Metaprogramming (when/when not) +- ActiveRecord usage +- Gem dependencies +- RuboCop compliance + +**Common Issues**: +- Fighting Rails conventions +- N+1 queries in ActiveRecord +- Overuse of metaprogramming +- Callback hell in models +- Fat controllers + +--- + +### Go Expert + +**Focus**: Idiomatic Go, concurrency, simplicity + +**Key Areas**: +- Goroutines and channels +- Error handling (not panic) +- Interface design +- Context usage +- Standard library preference +- Concurrency patterns + +**Common Issues**: +- Goroutine leaks +- Ignoring errors +- Overuse of interfaces +- Improper context propagation +- Not following Go conventions + +--- + +### Rust Expert + +**Focus**: Ownership/borrowing, memory safety, zero-cost abstractions + +**Key Areas**: +- Ownership rules +- Lifetime annotations +- Error handling (Result) +- Unsafe code justification +- Async Rust patterns +- Zero-copy optimizations + +**Common Issues**: +- Fighting the borrow checker +- Unnecessary clones +- Unsafe code without documentation +- Blocking in async functions +- Not leveraging zero-cost abstractions + +--- + +## Creating New Specialists + +When a specialist doesn't exist in `.architecture/members.yml`: + +### Step 1: Define the Specialist + +Create a new member entry: + +```yaml +- id: [specialist_id] # e.g., graphql_specialist + name: "[Person Name]" + title: "[Specialist Title]" + specialties: + - "[Primary specialty]" + - "[Secondary specialty]" + - "[Tertiary specialty]" + disciplines: + - "[Discipline 1]" + - "[Discipline 2]" + skillsets: + - "[Skill 1]" + - "[Skill 2]" + domains: + - "[Domain 1]" + - "[Domain 2]" + perspective: "[One sentence describing their unique viewpoint]" +``` + +### Step 2: Define Their Focus + +Document what this specialist reviews: +- Primary focus areas +- Key concerns +- Review checklist +- Common issues +- Best practices to reference + +### Step 3: Add to Team + +Update `.architecture/members.yml` with the new specialist. + +Inform user: "I've added [Name] ([Title]) to your architecture team." + +--- + +## Review Quality Guidelines + +### Excellent Reviews Include + +1. **Specific Issues**: File paths, line numbers, code snippets +2. **Impact Analysis**: Why it matters, what breaks if not fixed +3. **Concrete Solutions**: Exact recommendations with code examples +4. **Effort Estimates**: Small/Medium/Large for each recommendation +5. **Priority**: Critical/High/Medium/Low severity +6. **Context**: Industry standards, best practices cited +7. **Trade-offs**: Acknowledge when recommendations have costs + +### Avoid + +1. **Vague Concerns**: "Code is messy" → Be specific +2. **Out of Domain**: Security specialist shouldn't review performance +3. **Surface-Level**: Provide deep expert-level insights +4. **No Solutions**: Every concern needs actionable fix +5. **Missing Context**: Explain why, not just what + +--- + +## Reference Templates + +For the complete review document structure, see [../assets/specialist-review-template.md](../assets/specialist-review-template.md). + +For member YAML format, see the setup-architect skill's member template. diff --git a/.architecture/config.yml b/.architecture/config.yml new file mode 100644 index 00000000..c0e157aa --- /dev/null +++ b/.architecture/config.yml @@ -0,0 +1,393 @@ +# AI Software Architect Configuration +# This file controls operational modes and behavior of the architectural review framework + +# ============================================================================== +# PRAGMATIC GUARD MODE +# ============================================================================== +# The Pragmatic Guard Mode adds a specialized "Pragmatic Enforcer" architect +# who actively challenges complexity, questions abstractions, and pushes for +# the simplest solutions that meet current requirements. +# +# Enable this mode to guard against over-engineering and ensure YAGNI +# (You Aren't Gonna Need It) principles are applied rigorously. + +pragmatic_mode: + # Enable or disable Pragmatic Guard Mode + # Default: false (opt-in feature) + enabled: false + + # Intensity level controls how aggressively the Pragmatic Enforcer challenges + # complexity and pushes for simplicity + # + # - strict: Challenges aggressively, requires strong justification for any complexity + # Questions every "should" and "could", pushes for absolute minimal implementation + # + # - balanced: Challenges thoughtfully, accepts justified complexity (RECOMMENDED) + # Seeks middle ground between simplicity and best practices + # Questions "should" but accepts reasonable "must" + # + # - lenient: Raises concerns without blocking, suggests simpler alternatives as options + # Focuses on major complexity additions only + # Questions significant departures from simplicity + # + # Default: balanced + intensity: balanced + + # Control which review phases include the Pragmatic Enforcer + apply_to: + # Include in individual architect reviews + individual_reviews: true + + # Include in collaborative discussion phase + collaborative_discussions: true + + # Include when planning implementation steps + implementation_planning: true + + # Include during ADR (Architecture Decision Record) creation + adr_creation: true + + # Include in specific architect reviews (when user asks single architect) + specific_reviews: true + + # Exemptions: Areas where strict best practices should be maintained + # regardless of pragmatic mode intensity. The Pragmatic Enforcer will + # still participate but will not challenge security/compliance requirements. + exemptions: + # Never compromise on security-critical features + # Examples: authentication, authorization, encryption, input validation + security_critical: true + + # Never compromise on data integrity + # Examples: database transactions, data validation, backup strategies + data_integrity: true + + # Never compromise on compliance requirements + # Examples: GDPR, HIPAA, PCI-DSS, audit logging + compliance_required: true + + # Never compromise on accessibility requirements + # Examples: WCAG compliance, screen reader support + accessibility: true + + # Triggers: Situations that activate pragmatic analysis and challenges + # Disable specific triggers if you want to skip analysis for those cases + triggers: + # Challenge when adding new abstraction layers + # Examples: new interfaces, abstract base classes, dependency injection containers + new_abstraction_layer: true + + # Challenge when adding new external dependencies + # Examples: new libraries, frameworks, services + new_dependency: true + + # Challenge when introducing new architectural patterns + # Examples: repository pattern, strategy pattern, observer pattern + new_pattern_introduction: true + + # Challenge when expanding scope beyond initial requirements + # Examples: adding "nice to have" features, building for imagined future needs + scope_expansion: true + + # Challenge performance optimizations without evidence of problems + # Examples: caching layers, database indexes, query optimization + performance_optimization: true + + # Challenge comprehensive test infrastructure upfront + # Examples: full integration tests, E2E tests, property tests before code exists + test_infrastructure: true + + # Challenge adding flexibility or configurability + # Examples: feature flags, plugin systems, configuration frameworks + flexibility_addition: true + + # Thresholds: Numerical values that help determine when to challenge + thresholds: + # Minimum complexity score to trigger a challenge (0-10) + # Complexity is assessed by: abstraction layers, file count, dependencies, LOC + # Default: 5 (challenge moderate to high complexity) + min_complexity_score: 5 + + # Minimum necessity score to accept without challenge (0-10) + # Necessity is assessed by: current requirements, immediate value, cost of deferral + # Default: 7 (must be clearly necessary to skip challenge) + min_necessity_score: 7 + + # Maximum acceptable complexity-to-necessity ratio + # If complexity score / necessity score > threshold, challenge strongly + # Default: 1.5 (complexity should not exceed necessity by more than 50%) + max_complexity_ratio: 1.5 + + # Behavioral configuration + behavior: + # Require explicit justification for complexity + # When true, architects must proactively justify complex solutions + require_justification: true + + # Always propose simpler alternative + # When true, Pragmatic Enforcer always suggests a simpler approach + always_propose_alternative: true + + # Calculate and show cost of waiting + # When true, includes analysis of what happens if implementation is deferred + show_cost_of_waiting: true + + # Track deferred decisions + # When true, maintains a log of decisions deferred for future implementation + track_deferrals: true + + # Deferral log location + deferral_log: .architecture/deferrals.md + + # Question templates: Customize the questions the Pragmatic Enforcer asks + # Leave empty to use defaults + custom_questions: + necessity: [] + # - "Do we need this right now?" + # - "What breaks if we don't implement this?" + + simplicity: [] + # - "What's the simplest thing that could work?" + # - "Can we do this with less code?" + + cost: [] + # - "What's the cost of implementing this now?" + # - "What's the cost of waiting?" + + alternatives: [] + # - "What if we just...?" + # - "Could we use an existing tool?" + + best_practices: [] + # - "Does this best practice apply to our context?" + # - "Is this over-engineering for our scale?" + +# ============================================================================== +# IMPLEMENTATION GUIDANCE +# ============================================================================== +# Configure how AI assistants implement features when you use the command: +# "Implement X as the architects" +# +# This allows you to specify your development methodology, coding influences, +# and practices once, and have them applied consistently across all implementations. +# +# Benefits: +# - 90% reduction in prompt length (no more repeating preferences) +# - Consistent application of best practices +# - Team standards documented in version control +# - Better onboarding (new developers see documented practices) + +implementation: + # Enable or disable implementation guidance + # When enabled, "Implement X as the architects" will apply your configuration + # When disabled, implementations proceed without special guidance + # Default: true + enabled: true + + # Primary development methodology + # Specifies the overall approach to writing code + # Options: TDD, BDD, DDD, Test-Last, Exploratory, or custom + # + # - TDD (Test-Driven Development): Write tests first, red-green-refactor + # - BDD (Behavior-Driven Development): Behavior-focused tests, outside-in + # - DDD (Domain-Driven Design): Domain modeling, bounded contexts + # - Test-Last: Implementation first, tests after + # - Exploratory: Experiment, then codify + # + # Default: TDD + methodology: "TDD" + + # Coding influences - thought leaders whose practices to follow + # List specific books, talks, or articles that guide your approach + # + # Format: "Name - Source (focus area)" or just "Name - Source" + # + # Examples: + # - "Kent Beck - TDD by Example" + # - "Sandi Metz - POODR, 99 Bottles" + # - "Martin Fowler - Refactoring" + # - "Gary Bernhardt - Destroy All Software" + # - "Eric Evans - Domain-Driven Design" + # + # Can be organized as simple list or grouped by focus area + influences: + - "Kent Beck - TDD by Example (methodology fundamentals)" + - "Sandi Metz - POODR, 99 Bottles (OO design and refactoring)" + - "Martin Fowler - Refactoring (patterns and code smells)" + # Add more influences as needed + # - "Your favorite author - Their book/source" + + # Language-specific practices (optional) + # Customize for your project's primary language(s) + # Add sections for each language you use + languages: + # Example: Ruby configuration + # ruby: + # style_guide: "Rubocop" + # idioms: "Prefer blocks over loops, use meaningful method names" + # frameworks: + # rails: "Follow conventions but question them" + # sinatra: "Keep it simple" + + # Example: JavaScript/TypeScript configuration + # javascript: + # style_guide: "ESLint with Airbnb config" + # idioms: "Prefer const, use arrow functions, destructure" + # frameworks: + # react: "Functional components, hooks" + # vue: "Composition API" + + # Example: Python configuration + # python: + # style_guide: "PEP 8 via Black formatter" + # idioms: "Pythonic patterns, list comprehensions" + # frameworks: + # django: "Fat models, thin views" + # flask: "Keep it minimal" + + # Testing approach (optional) + # Specify your testing framework, style, and goals + testing: + # Testing framework to use + # framework: "RSpec" # or Minitest, Jest, pytest, JUnit, etc. + + # Testing style + # style: "Outside-in TDD" # or Inside-out, Detroit school, London school + + # Testing approach + # approach: "Mock judiciously, prefer real objects when practical" + + # Coverage goals + # coverage: "High for business logic, integration tests for critical paths" + + # Test speed targets + # speed: "Unit tests <100ms, integration tests <1s" + + # Refactoring guidelines (optional) + # When and how to refactor code + refactoring: + # When to refactor + # when: + # - "After tests green (red-green-REFACTOR)" + # - "When code smells emerge (long methods, duplication)" + # - "Rule of Three: refactor on third occurrence" + + # Refactoring principles + # principles: + # - "Small methods (≤5 lines per Sandi Metz)" + # - "Clear names over comments" + # - "Simple design over clever code" + + # Quality standards (optional) + # Define what "done" means for your team + quality: + # Definition of done checklist + # definition_of_done: + # - "Tests passing (all green)" + # - "Code refactored for clarity" + # - "No obvious code smells" + # - "Code reviewed by team" + # - "Documentation updated (if public API)" + + # Quality priorities (in order) + # priorities: + # - "Clarity first (code is read more than written)" + # - "Simplicity second (avoid premature abstraction)" + # - "Performance third (optimize when measured need exists)" + + # Security practices (optional but recommended) + # These are always applied, even with pragmatic_mode enabled + # Security practices are exempt from YAGNI challenges + security: + # Mandatory security practices + # Always applied regardless of other settings + # mandatory_practices: + # - "Input validation (whitelist approach)" + # - "Output encoding (context-aware)" + # - "Parameterized queries (no string concatenation)" + # - "Authentication checks (verify identity)" + # - "Authorization checks (verify permissions)" + + # Security influences + # influences: + # - "OWASP Secure Coding Practices" + # - "SEI CERT Coding Standards" + + # Performance considerations (optional) + # Configure performance-aware implementation + performance: + # Is this a performance-critical system? + # If true, adds performance-focused practices and influences + # critical: false + + # Performance practices + # practices: + # - "Profile before optimizing" + # - "Benchmark critical paths" + # - "Set performance budgets" + + # Performance influences (when critical: true) + # influences: + # - "Brendan Gregg - Systems Performance" + +# ============================================================================== +# GENERAL CONFIGURATION +# ============================================================================== + +# Review process configuration +review_process: + # Require all members to review before collaborative phase + require_all_members: true + + # Maximum time for individual review phase (for planning purposes) + max_individual_phase_days: 3 + + # Maximum time for collaborative discussion (for planning purposes) + max_collaborative_phase_days: 2 + +# Architecture Decision Record (ADR) configuration +adr: + # Numbering format: sequential | date-based + numbering_format: sequential + + # Require alternatives section + require_alternatives: true + + # Require validation criteria + require_validation: true + +# Member configuration +members: + # Path to members.yml file (relative to .architecture) + definition_file: members.yml + + # Allow dynamic member creation + # When true, new members can be added during reviews if expertise is needed + allow_dynamic_creation: true + +# Templates configuration +templates: + # Directory containing templates (relative to .architecture) + directory: templates + + # Auto-populate template metadata + auto_populate_metadata: true + +# Output configuration +output: + # Verbosity level: minimal | normal | detailed + verbosity: normal + + # Include reasoning in outputs + include_reasoning: true + + # Format: markdown | json | yaml + format: markdown + +# Version tracking +version: + # Current framework version + framework_version: "1.2.0" + + # Project architecture version (update with each major architecture change) + architecture_version: "1.0.0" diff --git a/.architecture/decisions/adrs/ADR-001-reorganize-repository-structure-for-core-and-full-installations.md b/.architecture/decisions/adrs/ADR-001-reorganize-repository-structure-for-core-and-full-installations.md new file mode 100644 index 00000000..3d4c4711 --- /dev/null +++ b/.architecture/decisions/adrs/ADR-001-reorganize-repository-structure-for-core-and-full-installations.md @@ -0,0 +1,219 @@ +# ADR-001: Reorganize Repository Structure using Split Directories for Core and Full Installations + +## Status + +Proposed + +## Context + +The `opl` library has grown to include many tools and modules, some of which require heavy or environment-specific dependencies (e.g., `psycopg2-binary`, `kafka-python`, `locust`). However, many consumers of this repository only need a small "core" subset of tools (e.g., `junit_cli.py`, `cluster_read.py`) that run with a minimal dependency footprint. + +Historically, this was solved by maintaining a separate copy of the core Python modules inside the `core/opl/` directory, with its own `core/setup.py`. This leads to code duplication, increased maintenance burden, and the risk of the core and full modules drifting out of sync. We need a way to maintain a single source of truth for the codebase while continuing to support both a full installation and a lightweight core installation directly from the git repository using `pip`. + +An initial proposal involved pointing `core/setup.py` back to the root `opl/` directory to share files. However, this relies on non-standard packaging paths (`../`) and fails to enforce a hard boundary preventing developers from accidentally adding heavy dependencies to core modules. + +## Decision Drivers + +* Code duplication between `opl/` and `core/opl/` increases maintenance burden and risk of bugs. +* Consumers need to be able to install just the core functionality without pulling in heavy dependencies. +* The installation commands for consumers (using `pip install git+https...`) are established and should ideally remain the same, specifically utilizing the `--subdirectory=core` feature of pip for the core installation. +* We want to physically enforce a hard boundary between core functionality and tools that require heavy dependencies. +* Python packaging constraints should be respected, avoiding hacks like referencing parent directories in `setup.py`. + +## Decision + +We will reorganize the repository structure into a **Split Directories** pattern, creating a workspace/monorepo style layout using Python's implicit namespace packages (PEP 420). + +**Architectural Components Affected:** +* **Directory Structure:** The existing `opl/` directory will be split into two physically separate directories: `core/opl/` and `extras/opl/`. +* **Namespace Packaging:** `opl/__init__.py` files will be removed or converted to namespace packages to allow `import opl.*` to resolve across both physical directories seamlessly. +* **Packaging Configuration:** + * `core/setup.py`: Standard setup file specifying only minimal dependencies. + * `extras/setup.py`: Standard setup file specifying heavy dependencies. + * Root `setup.py`: Acts as a "meta-package" that installs both `core` and `extras` via local file references (`install_requires=[f"opl-core @ file://{os.path.abspath('core')}", ...]`). + +This architecture completely eliminates duplication while physically isolating core modules from heavy dependencies. + +## Consequences + +### Positive + +* Single source of truth for all code: bug fixes and features only need to be implemented once. +* Hard physical boundaries: It is impossible to accidentally import a heavy dependency into a core module. +* Clean Packaging: `core/setup.py` doesn't need to use any `../` hacks. +* True Modularity: Aligns perfectly with modern Python workspace/monorepo patterns. +* Preserves Install Commands: Both `pip install git+https://...` and `pip install git+https://...#subdirectory=core` will continue to function seamlessly. + +### Negative + +* Initial Refactor Effort: Requires identifying every file currently in `opl/` and explicitly sorting them into either `core/opl/` or `extras/opl/`. +* Testing complexity: pytest configuration might need tweaking to discover tests across multiple source directories or ensure the implicit namespace package is correctly loaded during test runs. + +### Neutral + +* The root directory no longer contains source code directly; it merely orchestrates the sub-packages. + +## Implementation Strategy + +> **Note**: This section externalizes the "senior thinking" about HOW and WHEN to implement this decision, not just WHAT to implement. + +### Blast Radius + +**Impact Scope**: Local development environments, CI pipelines, and external consumers relying on git-based pip installs. + +**Affected Components**: +- Packaging configuration (Root `setup.py`, `core/setup.py`, `extras/setup.py`) +- Python module structure (`opl/` -> `core/opl/` & `extras/opl/`) +- CI testing scripts (pytest configurations) + +**Affected Teams**: +- All teams consuming the `opl` repository via pip. + +**User Impact**: Users installing via pip from the git URL should experience no change in functionality, but the core installation will now correctly reflect the unified code. Imports like `from opl.module import foo` will continue working exactly as before thanks to namespace packaging. + +**Risk Mitigation**: +- Exhaustively test `pip install` commands locally and in a clean container before merging. +- Verify that `pytest` can successfully discover and run tests for all modules. +- Ensure that the entry points defined in `core/setup.py` and the root `setup.py` still function correctly. + +### Reversibility + +**Reversibility Level**: High + +**Rollback Feasibility**: +- Very easy to rollback by simply reverting the git commit if issues are discovered post-merge. + +**Migration Paths**: +- **Forward Migration**: Split the `opl/` files into `core/opl/` and `extras/opl/`. Setup the meta-package `setup.py` in the root. Verify installation and tests. +- **Rollback Migration**: Revert the commit. +- **Evolution Path**: If the implicit namespace package proves too brittle across varying Python environments, we can investigate explicit `pkgutil` style namespace declarations. + +**Options Preserved**: +- We can still extract core to a separate repository in the future if needed, and it would be significantly easier to do so since it is already physically isolated. + +**Commitments Made**: +- We commit to organizing code into these logical groupings (`core` vs `extras`) and maintaining the namespace package structure. + +### Sequencing & Timing + +**Prerequisites**: +- [ ] Technical spike completed successfully to validate the meta-package approach with Git URL installs (Done). +- [ ] Identify which files in `opl/` belong in `core/` and which belong in `extras/`. + +**System Readiness**: +- **Dependencies**: Consumers' environments are unaffected as long as the install command continues to work. + +**Team Readiness**: +- **Understanding**: The team needs to understand the new directory layout and that all new modules must be placed in the appropriate sub-directory (`core` or `extras`). + +**Sequencing Concerns**: +- None. This is an internal repository refactoring. + +**Readiness Assessment**: Ready to implement. + +### Social Cost + +**Learning Curve**: Low +- Developers just need to remember to navigate into `core/opl/` or `extras/opl/` instead of a top-level `opl/` directory. + +**Cognitive Load**: +- Reduced, as boundaries between lightweight and heavy tools are now physically enforced. + +**Clarity Assessment**: +- **Will this help more than confuse?**: Yes. Duplication is inherently confusing, and clear physical boundaries prevent architectural degradation. +- **Explanation required**: A brief team announcement regarding the directory structure change. + +**Documentation Needs**: +- [ ] Update any developer documentation that mentions the location of source files. + +### Confidence Assessment + +**Model Correctness Confidence**: High +- The approach leverages standard Python PEP 420 namespace packaging and standard pip install mechanisms. + +**Assumptions**: +1. Python's implicit namespace packages (PEP 420) work consistently across all our supported Python versions (>=3.6). - **Validation**: Requires verifying behavior in Python 3.6+ environments, particularly regarding `__init__.py` files. + +**Uncertainty Areas**: +- Ensuring all CI testing tools (like `pytest`, `flake8`, `mypy`) correctly handle the split directories and namespace packages without significant configuration changes. + +**Validation Approach**: +- Run the full CI test suite against the new branch locally. + +## Implementation + +**Phase 1: Structure Preparation (Completed)** +* Create `extras/opl/` directory. +* Remove `opl/__init__.py` (if it doesn't contain vital code) to enable implicit namespace packaging, or convert it to a `pkgutil` style namespace declaration if needed for compatibility. + +**Phase 2: Code Migration (Completed)** +* Move heavy-dependency tools and modules from `opl/` into `extras/opl/`. +* Move the remaining lightweight core tools from `opl/` into `core/opl/`. +* Delete the now-empty root `opl/` directory. + +**Phase 3: Packaging Update (Completed)** +* Update `core/setup.py`. +* Create `extras/setup.py`. +* Update the root `setup.py` to act as the meta-package pulling in both local directories. + +**Phase 4: Validation & Cleanup (Completed)** +* Run tests to ensure imports resolve correctly. +* Update `tox.ini`, GitHub Actions, or any other CI configuration that hardcodes the `opl/` path. +* Open a Pull Request for review. + +## Alternatives Considered + +### Package Core and use Extras for Full + +Move all code to a single package, and use `extras_require` for the heavy dependencies (e.g., `pip install .[full]`). + +**Pros:** +* Standard Python packaging practice. +* Single `setup.py` file. + +**Cons:** +* Breaks backwards compatibility with the existing `subdirectory=core` installation method. +* Users would have to change their CI/CD pipelines to update the pip install command. +* Doesn't physically enforce the boundary between core code and heavy code. + +### Extract Core to a Separate Repository + +Move the core tools to an entirely new repository (`opl-core`) and have the main `opl` repository depend on it. + +**Pros:** +* Hardest boundary prevents accidental dependency leakage. +* Clean separation of concerns. + +**Cons:** +* Significant overhead to create and maintain a new repository. +* Requires coordinating changes across two repositories. + +### Single source of truth using `../` references + +Point `core/setup.py` back to the root `opl/` directory (`package_dir={'': '..'}`). + +**Pros:** +* Very simple change to `setup.py`. +* No need to split directories. + +**Cons:** +* Relies on non-standard `../` references in packaging which can break certain build tools. +* Fails to physically separate core modules from heavy modules, relying purely on developer discipline to avoid bad imports. + +## Validation + +**Acceptance Criteria:** +- [ ] The `opl/` directory is successfully split into `core/opl/` and `extras/opl/`. +- [ ] No duplicated Python files exist in the repository. +- [ ] Running `python -m pip install git+https://github.com/redhat-performance/opl.git` succeeds, installs the full toolset (both core and extras), and imports work seamlessly. +- [ ] Running `python3 -m pip install --no-cache-dir -e "git+https://github.com/redhat-performance/opl.git#egg=opl-rhcloud-perf-team-core&subdirectory=core"` succeeds, installs only the core tools, and those tools are executable. +- [ ] All automated tests pass successfully. + +**Testing Approach:** +* Use a clean Python virtual environment to test both installation commands sequentially. +* Run the existing `pytest` suite locally against the new directory structure. + +## References + +* [PEP 420 -- Implicit Namespace Packages](https://peps.python.org/pep-0420/) +* Setuptools Documentation on Package Discovery \ No newline at end of file diff --git a/.architecture/deferrals.md b/.architecture/deferrals.md new file mode 100644 index 00000000..2e0aa0bf --- /dev/null +++ b/.architecture/deferrals.md @@ -0,0 +1,106 @@ +# Deferred Architectural Decisions + +This document tracks architectural features, patterns, and complexity that have been consciously deferred for future implementation. Each deferral includes the rationale for waiting and clear trigger conditions for when it should be reconsidered. + +## Status Key + +- **Deferred**: Decision to defer is active, watching for trigger +- **Triggered**: Trigger condition met, needs implementation +- **Implemented**: Feature has been implemented (moved to ADR) +- **Cancelled**: No longer needed or relevant + +--- + +## Deferred Decisions + +*(No deferred decisions currently tracked)* + +--- + +## Review Process + +This document should be reviewed: + +**Monthly**: Check for triggered conditions +- Review each deferred item +- Check if any trigger conditions have been met +- Update priority if context has changed + +**Quarterly**: Re-evaluate deferrals +- Are deferred items still relevant? +- Have requirements changed? +- Should priority be adjusted? +- Can any be cancelled? + +**During Architecture Reviews**: Reference deferrals +- Check if new features relate to deferrals +- Consider if triggered conditions met +- Avoid re-proposing already-deferred items +- Update relevant entries + +**When Triggers Met**: +1. Update status to "Triggered" +2. Create or update ADR for implementation +3. Plan implementation in next sprint/release +4. Reference this deferral in the ADR +5. After implementation, update status to "Implemented" + +## Metrics + +Track deferral outcomes to improve decision-making: + +| Metric | Value | Notes | +|--------|-------|-------| +| Total deferrals | 0 | All-time count | +| Active deferrals | 0 | Currently deferred | +| Triggered awaiting implementation | 0 | Need to address | +| Implemented | 0 | Were eventually needed | +| Cancelled | 0 | Were never needed | +| Average time before trigger | - | How long before we needed it | +| Hit rate (implemented/total) | 0% | How often deferred things are needed | + +**Target**: < 40% hit rate (most deferred things remain unneeded, validating deferral decisions) + +--- + +## Template for New Deferrals + +When adding a deferral, use this format: + +```markdown +### [Feature/Pattern Name] + +**Status**: Deferred +**Deferred Date**: YYYY-MM-DD +**Category**: [Architecture | Performance | Testing | Infrastructure | Security] +**Priority**: [Low | Medium | High] + +**What Was Deferred**: +[Brief description of the feature, pattern, or complexity that was deferred] + +**Original Proposal**: +[What was originally suggested - can quote from review or ADR] + +**Rationale for Deferring**: +- Current need score: [0-10] +- Complexity score: [0-10] +- Cost of waiting: [Low | Medium | High] +- Why deferring makes sense: [Explanation] + +**Simpler Current Approach**: +[What we're doing instead for now] + +**Trigger Conditions** (Implement when): +- [ ] [Specific condition 1 - make this measurable] +- [ ] [Specific condition 2] +- [ ] [Specific condition 3] + +**Implementation Notes**: +[Notes for when this is implemented - gotchas, considerations, references] + +**Related Documents**: +- [Link to ADR or review] +- [Link to discussion] + +**Last Reviewed**: YYYY-MM-DD +``` diff --git a/.architecture/members.yml b/.architecture/members.yml new file mode 100644 index 00000000..a2fb43bc --- /dev/null +++ b/.architecture/members.yml @@ -0,0 +1,234 @@ +# Architecture Review Members +# Each member represents a specialized perspective for architectural reviews + +members: + - id: systems_architect + name: "Systems Architect" + title: "Systems Architect" + specialties: + - "distributed systems" + - "service architecture" + - "scalability patterns" + disciplines: + - "system design" + - "infrastructure modeling" + - "load distribution" + skillsets: + - "high-level architecture planning" + - "system decomposition" + - "technical roadmapping" + domains: + - "overall system coherence" + - "architectural principles" + - "technical governance" + perspective: "Focuses on how components work together as a cohesive system and analyzes big-picture architectural concerns." + + - id: domain_expert + name: "Domain Expert" + title: "Domain Expert" + specialties: + - "domain-driven design" + - "bounded contexts" + - "ubiquitous language" + disciplines: + - "business logic modeling" + - "domain analysis" + - "requirements engineering" + skillsets: + - "translating business needs to technical solutions" + - "identifying core domain concepts" + - "defining domain boundaries" + domains: + - "business logic representation" + - "semantic accuracy" + - "model integrity" + perspective: "Evaluates how well the architecture represents and serves the problem domain and business concepts." + + - id: security_specialist + name: "Security Specialist" + title: "Security Specialist" + specialties: + - "threat modeling" + - "security patterns" + - "data protection" + disciplines: + - "security analysis" + - "risk assessment" + - "vulnerability identification" + skillsets: + - "identifying security implications" + - "recommending security controls" + - "evaluating authentication/authorization approaches" + domains: + - "system security" + - "data protection" + - "access control" + perspective: "Reviews the architecture from a security-first perspective, identifying potential vulnerabilities and security implications." + + - id: maintainability_expert + name: "Maintainability Expert" + title: "Maintainability Expert" + specialties: + - "code quality" + - "refactoring" + - "technical debt" + disciplines: + - "code organization" + - "complexity analysis" + - "maintainability metrics" + skillsets: + - "identifying maintenance bottlenecks" + - "assessing technical debt" + - "simplifying complex structures" + domains: + - "code maintainability" + - "architectural clarity" + - "evolutionary design" + perspective: "Evaluates how well the architecture facilitates long-term maintenance, evolution, and developer understanding." + + - id: performance_specialist + name: "Performance Specialist" + title: "Performance Specialist" + specialties: + - "performance optimization" + - "resource utilization" + - "scalability planning" + disciplines: + - "performance analysis" + - "bottleneck identification" + - "efficiency improvement" + skillsets: + - "profiling system performance" + - "identifying performance critical paths" + - "optimizing resource usage" + domains: + - "system performance" + - "response time" + - "throughput" + perspective: "Focuses on performance implications of architectural decisions and suggests optimizations." + + - id: implementation_strategist + name: "Implementation Strategist" + title: "Implementation Strategist" + specialties: + - "change sequencing" + - "blast radius analysis" + - "reversibility design" + - "team readiness assessment" + - "migration planning" + disciplines: + - "change management" + - "risk analysis" + - "phased rollouts" + - "impact assessment" + - "timing optimization" + skillsets: + - "evaluating change impact scope" + - "designing reversible architectures" + - "assessing system and team readiness" + - "sequencing dependent changes" + - "calculating social cost of changes" + domains: + - "implementation timing" + - "change risk management" + - "architectural evolution" + - "team capability alignment" + perspective: "Evaluates HOW and WHEN changes should be implemented, considering blast radius, reversibility, technical readiness, team capability, and timing. Asks 'Is the system ready?' and 'Is the team ready?' before 'Can we build it?'" + + - id: ai_engineer + name: "AI Engineer" + title: "AI Engineer" + specialties: + - "AI/ML integration" + - "LLM application design" + - "agent-based systems" + - "AI evaluation frameworks" + - "agent orchestration" + disciplines: + - "applied AI" + - "prompt engineering" + - "agent workflow design" + - "AI system observability" + - "multi-agent coordination" + skillsets: + - "building practical AI solutions" + - "evaluating user experience with AI" + - "integrating AI components" + - "designing AI metrics and benchmarks" + - "implementing observability systems" + domains: + - "end-user AI applications" + - "AI development workflows" + - "practical implementation" + - "AI system evaluation" + - "observability and metrics" + perspective: "Evaluates the architecture from an AI integration perspective, focusing on practical utility, system evaluation, observability, and agent interaction patterns." + + - id: pragmatic_enforcer + name: "Pragmatic Enforcer" + title: "YAGNI Guardian & Simplicity Advocate" + specialties: + - "YAGNI principles" + - "incremental design" + - "complexity analysis" + - "requirement validation" + - "minimum viable solutions" + disciplines: + - "scope management" + - "cost-benefit analysis" + - "technical debt prevention" + - "simplification strategies" + - "deferral decision-making" + skillsets: + - "identifying premature optimization" + - "challenging unnecessary abstractions" + - "proposing simpler alternatives" + - "calculating cost of waiting" + - "questioning best-practice applicability" + domains: + - "implementation simplicity" + - "requirement sufficiency" + - "appropriate complexity" + perspective: "Rigorously questions whether proposed solutions, abstractions, and features are actually needed right now, pushing for the simplest approach that solves the immediate problem." + mode_specific: + active_when: "pragmatic_mode.enabled == true" + tunable: true + default_phases: + - "individual_reviews" + - "collaborative_discussions" + - "implementation_planning" + - "adr_creation" + + - id: python_expert + name: "Dr. Sarah Chen" + title: "Python Expert" + specialties: + - "Python best practices" + - "Package architecture" + - "Type hints and mypy" + disciplines: + - "Pythonic code" + - "Testing patterns" + - "Performance optimization" + skillsets: + - "Modern Python (3.10+)" + - "Async/await" + - "Data structures" + domains: + - "Backend services" + - "Data processing" + - "API development" + perspective: "Advocates for clean, Pythonic solutions following PEP guidelines" + +review_process: + individual_phase: + description: "Each member reviews the architecture independently, focusing on their area of expertise." + deliverable: "Individual member review section in the review document." + + collaborative_phase: + description: "Members confer to discuss findings, resolve conflicting perspectives, and prioritize recommendations." + deliverable: "Consolidated findings and balanced recommendations." + + final_report_phase: + description: "Produce a comprehensive architectural review that balances all perspectives." + deliverable: "Final architectural review document with actionable recommendations." \ No newline at end of file diff --git a/.architecture/principles.md b/.architecture/principles.md new file mode 100644 index 00000000..bbe74e8d --- /dev/null +++ b/.architecture/principles.md @@ -0,0 +1,275 @@ +# Architectural Principles + +This document outlines the core architectural principles that guide all design decisions in the project. These principles should be referenced when making architectural decisions, evaluating designs, and conducting reviews. + +## Wisdom From Software Pioneers + +> "Any fool can write code that a computer can understand. Good programmers write code that humans can understand." +> — **Martin Fowler** + +> "Bad programmers worry about the code. Good programmers worry about data structures and their relationships." +> — **Linus Torvalds** + +> "The heart of software is its ability to solve domain-related problems for its user." +> — **Eric Evans** + +> "I'm not a great programmer; I'm just a good programmer with great habits." +> — **Kent Beck** + +> "Adding manpower to a late software project makes it later." +> — **Fred Brooks** + +> "Clean code is simple and direct. Clean code reads like well-written prose." +> — **Robert C. Martin** + +> "Do The Simplest Thing That Could Possibly Work" +> — **Kent Beck** + +> "You don't send messages because you have objects, you have objects because you send messages." +> — **Sandi Metz** + +> "Your application needs to work right now just once; it must be easy to change forever." +> — **Sandi Metz** + +> "The modern practice of software isn't much like architecture or construction. The buildings are largely built. These days, we make a pre-built space work for whoever lives there." +> — **Sarah Mei** + +## Core Principles + +### 0. Livable Code + +**Description**: Software should be designed for the people who need to live in it, not just for architectural purity or technical elegance. + +**Rationale**: Developers spend most of their time working with existing code. A codebase should be comfortable to inhabit, maintain, and evolve over time. + +**Wisdom**: +> "Is your code the kind of cluttered house you might find on a reality TV show? Or the kind of sleek, minimalist house you might find in an architectural magazine? Neither one sounds like a place you could comfortably live." +> — **Sarah Mei** + +**Application**: +- Design for the day-to-day experience of developers working in the codebase +- Balance technical purity with practical needs +- Consider the cognitive load of navigating the system +- Focus on making the codebase welcoming to newcomers +- Debug communication patterns when code smells persist + +### 1. Clarity over Cleverness + +**Description**: Prefer simple, clear designs over clever, complex solutions, even when the latter might be more elegant or slightly more efficient. + +**Rationale**: Clear designs are easier to understand, maintain, extend, and debug. They reduce cognitive load and make onboarding new developers faster. + +**Wisdom**: +> "I define architecture as a word we use when we want to talk about design but want to puff it up to make it sound important." +> — **Martin Fowler** + +**Application**: +- Favor straightforward implementations over complex abstractions +- Use established patterns over novel approaches when possible +- Document the "why" behind design decisions, not just the "how" +- If you have to explain how it works, it's probably too complex + +### 2. Separation of Concerns + +**Description**: Systems should be divided into distinct features with minimal overlap in functionality. Each component should have a clear responsibility. + +**Rationale**: Well-separated components can be developed, tested, and maintained independently, reducing cognitive load and simplifying changes. + +**Wisdom**: +> "The Single Responsibility Principle. A Module should be responsible to one, and only one, actor." +> — **Robert C. Martin** + +**Application**: +- Define clear boundaries between subsystems +- Design components with single responsibilities +- Avoid shared state and hidden dependencies +- Use interfaces to decouple implementation details +- Focus on the domain model as the heart of your system + +### 3. Evolvability + +**Description**: The architecture should facilitate change and evolution over time without requiring complete rewrites. + +**Rationale**: Requirements always change, and the ability to adapt is essential for long-term sustainability. + +**Wisdom**: +> "Architecture represents the significant design decisions that shape a system, where significant is measured by cost of change." +> — **Grady Booch** + +> "The future is uncertain and you will never know less than you know right now." +> — **Sandi Metz** + +**Application**: +- Design for extensibility at key variation points +- Create stable interfaces between components +- Use abstraction layers to isolate changes +- Anticipate and plan for version transitions +- Create a model that gives the development team leverage against complexity +- Resist the urge to overdesign for imagined future needs + +### 4. Observability + +**Description**: The system should provide insights into its behavior, performance, and state. + +**Rationale**: Effective monitoring, debugging, and performance optimization require visibility into system operation. + +**Wisdom**: +> "Truth can only be found in one place: the code." +> — **Robert C. Martin** + +**Application**: +- Build in logging, metrics, and tracing from the start +- Make error states and edge cases explicit and observable +- Design components to expose health and performance data +- Create clear audit trails for important operations +- Favor explicit over implicit behavior + +### 5. Security by Design + +**Description**: Security must be an integral part of the architecture, not an afterthought. + +**Rationale**: Retrofitting security is more expensive, less effective, and often leaves vulnerabilities. + +**Wisdom**: +> "Code, without tests, is not clean. No matter how elegant it is, no matter how readable and accessible, if it hath not tests, it be unclean." +> — **Robert C. Martin** + +**Application**: +- Implement proper authentication and authorization at all levels +- Apply principle of least privilege +- Design for secure defaults +- Perform threat modeling during design phase +- Validate all inputs and sanitize all outputs + +### 6. Domain-Centric Design + +**Description**: The architecture should reflect and serve the problem domain, with a ubiquitous language that bridges technical and business concerns. + +**Rationale**: When the model and implementation are closely aligned with the business domain, the system is more likely to meet user needs and adapt to changes in requirements. + +**Wisdom**: +> "A model is a selectively simplified and consciously structured form of knowledge." +> — **Eric Evans** + +**Application**: +- Create a ubiquitous language shared by developers and domain experts +- Model the domain, not just the data +- Separate core domain from supporting subdomains +- Make implicit concepts explicit in the model +- Continuously refine the model as understanding deepens + +### 7. Pragmatic Simplicity + +**Description**: Favor practical, working solutions over theoretical perfection. Recognize that software architecture is an exercise in managing complexity, not eliminating it. + +**Rationale**: Theoretical purity often adds complexity without proportional benefit. Real systems must balance many competing concerns. + +**Wisdom**: +> "Theory and practice sometimes clash. And when that happens, theory loses. Every single time." +> — **Linus Torvalds** + +> "Instead of deciding on a class and then figuring out its responsibilities, you are now deciding on a message and figuring out where to send it. This transition from class-based design to message-based design is a turning point in your design career." +> — **Sandi Metz** + +**Application**: +- Value working software over comprehensive documentation +- Make incremental improvements rather than seeking perfection +- Question abstractions that don't demonstrably simplify the system +- Consider operational concerns alongside architectural purity +- Be willing to make pragmatic trade-offs when necessary +- Design from messages outward, not from classes inward +- When the future cost of doing nothing is the same as the current cost, postpone the decision + +### 8. Change Impact Awareness + +**Description**: Every architectural decision should explicitly consider its blast radius, reversibility, timing, and social cost before implementation. + +**Rationale**: Senior engineers instinctively evaluate these factors, but this "silent checklist" rarely gets documented. Making change impact explicit enables better decisions and knowledge sharing. + +**Wisdom**: +> "The act of explanation does something important. It slows you down enough to notice when your instincts are off. It also makes your reasoning visible in a way that solo work rarely does." +> — **Obie Fernandez** + +**Application**: +- **Blast Radius**: Identify the scope of impact if a change fails or needs reversal. Which components, teams, and users are affected? +- **Reversibility**: Design changes to be reversible when possible. Ask "If we're wrong, how hard is it to undo?" Prefer approaches that keep options open. +- **Sequencing & Timing**: Consider whether the system and team are ready for a change. A technically correct change can still be wrong if introduced at the wrong time. +- **Social Cost**: Evaluate whether a solution will confuse more people than it helps. Consider the learning curve and cognitive load on the team. +- **False Confidence**: Question whether tests passing actually validates the model is correct. Look for places where implementation correctness doesn't guarantee solution correctness. +- **Change Characterization**: Clarify whether you're introducing a new idea or expressing an existing one in a new place. Understand if changes belong at the surface or deep in the system. +- **Spread Analysis**: If this pattern or approach spreads across the codebase, is that desirable or a liability? + +**Examples**: + +*Good - High Impact Awareness:* +``` +Decision: Introduce new authentication pattern +Blast Radius: Affects 23 services, 4 teams, requires coordination +Reversibility: Medium - can rollback but requires migration scripts +Timing: System ready (observability in place), but Team B needs training +Social Cost: High learning curve, but improves long-term maintainability +Decision: Proceed with phased rollout, Team B training first +``` + +*Bad - Ignoring Impact:* +``` +Decision: Introduce new authentication pattern +Implementation: Changed all services in one release +Result: Outage across all services, no rollback plan, team confused +``` + +### 9. Pythonic Design + +**Description**: Follow established Python conventions and leverage the language's specific features and ecosystem effectively. + +**Rationale**: Idiomatic Python is more readable, maintainable, and performant. Fighting the language or importing idioms from other languages leads to friction. + +**Application**: +- **Explicit Over Implicit**: Make behavior clear, avoid magic, prefer explicit configuration. +- **Leverage the Standard Library**: Use built-in features (e.g., `collections`, `itertools`) before adding dependencies. +- **Type Hints**: Use type hints (`typing`) to document interfaces and enable static analysis (e.g., `mypy`), especially for public APIs and complex data structures. +- **Pythonic Code**: Use list comprehensions, generators, and context managers (`with` statements) where appropriate. Avoid C-style loops or getter/setter methods unless necessary. +- **Ecosystem Patterns**: Adhere to PEP 8 guidelines. + +## Implementation Guidelines + +### Documentation + +All significant architectural decisions should be documented in Architectural Decision Records (ADRs) that explain: +- The context and problem being addressed +- The decision that was made +- The consequences (positive and negative) +- Alternatives that were considered + +### Review Process + +Architectural changes should be reviewed against these principles. Reviews should explicitly address: +- How the change upholds or challenges each principle +- Trade-offs made between principles when they conflict +- Measures to mitigate any negative consequences + +### Exceptions + +Exceptions to these principles may be necessary in specific circumstances. When making exceptions: +- Document the exception and its justification +- Define the scope and boundaries of the exception +- Create a plan to eliminate or reduce the exception over time if possible +- Get approval from the architecture team + +## Evolution of Principles + +These principles should be reviewed and potentially revised: +- At the start of each major version development cycle +- When consistently encountering situations where the principles seem inadequate +- When new team members bring valuable alternative perspectives + +Proposed changes to principles should be discussed by the entire architecture team and require consensus to adopt. + +> "I will contend that conceptual integrity is the most important consideration in system design." +> — **Fred Brooks** + +> "Successful (working) but undesigned applications carry the seeds of their own destruction; they are easy to write, but gradually become impossible to change." +> — **Sandi Metz** + +> "By reflecting on code that's been written, and 'code smells' that keep coming up, you can determine missing links in communication. You could continue to solve those code smells. You could refactor them all away - once per sprint even! However, that's only addressing the symptoms. The underlying problem is more likely to be one of communication." +> — **Sarah Mei** \ No newline at end of file diff --git a/.architecture/reviews/ADR-001-split-directories.md b/.architecture/reviews/ADR-001-split-directories.md new file mode 100644 index 00000000..55436deb --- /dev/null +++ b/.architecture/reviews/ADR-001-split-directories.md @@ -0,0 +1,409 @@ +# Architecture Review: ADR-001 Reorganize Repository Structure + +**Date**: 2024-05-24 +**Review Type**: Feature / Architecture Change +**Reviewers**: Systems Architect, Domain Expert, Security Specialist, Maintainability Expert, Performance Specialist, Implementation Strategist, AI Engineer, Python Expert + +## Executive Summary + +This document reviews ADR-001, which proposes reorganizing the `opl` repository into a Split Directories pattern (using `core/opl/` and `extras/opl/`) with PEP 420 implicit namespace packages. The goal is to eliminate code duplication between core and full installations while enforcing a clean physical dependency boundary and maintaining existing `pip install` behaviors. + +**Overall Assessment**: Strong + +The proposed architecture provides a clean, modern Python approach to the problem. It completely removes the significant maintenance burden of duplicated code while elegantly enforcing physical boundaries to prevent heavy dependencies from leaking into the core toolset. + +**Key Findings**: +- The transition to PEP 420 namespace packages is the most idiomatic Python solution to this problem. +- The change is highly beneficial for maintainability by ensuring a single source of truth. +- The primary risks are related to developer tooling (pytest, mypy, linters) and the potential for severe merge conflicts during the transition. + +**Critical Actions**: +- Validate PEP 420 implicit namespace package compatibility with existing CI tooling (pytest, flake8) before finalizing the merge. +- Coordinate the implementation merge with the entire development team to minimize disruption to inflight pull requests. + +--- + +## System Overview + +- **Target**: ADR-001 - Reorganize Repository Structure for Core and Full Installations +- **Scope**: Repository directory structure, packaging configuration (`setup.py`), and Python module layout. +- **Key Technologies**: Python 3, `pip`, PEP 420 (Implicit Namespace Packages), `setuptools`. +- **Architecture Style**: Monorepo / Workspace layout with meta-packaging. +- **Context**: The repository currently duplicates core logic to offer a lightweight installation option without heavy dependencies. This ADR aims to unify the codebase while preserving the dual installation modes. + +--- + +## Individual Member Reviews + +### Systems Architect - Systems Architect + +**Perspective**: Focuses on how components work together as a cohesive system and analyzes big-picture architectural concerns. + +#### Key Observations +- Shifts the repository from a single standard package to a workspace-style layout with a meta-package. +- Leverages PEP 420 namespace packaging to unite physically distinct directories into a single logical Python namespace. + +#### Strengths +1. **Single Source of Truth**: Eliminates code duplication, inherently reducing bugs and architectural drift. +2. **Hard Physical Boundaries**: Structurally prevents the accidental import of heavy dependencies within core modules. + +#### Concerns +1. **Tooling Compatibility** (Impact: Medium) + - **Issue**: Some static analysis tools or legacy test runners might struggle with PEP 420 implicit namespaces. + - **Why it matters**: CI pipelines could fail or provide false negatives. + - **Recommendation**: Ensure CI exhaustively covers linters and type checkers against the new layout. + +#### Recommendations +1. **CI Pipeline Validation** (Priority: High, Effort: Small) + - **What**: Update all CI checks to ensure they run correctly across the split structure. + - **Why**: To prevent silent failures in test discovery. + - **How**: Run tests explicitly against both `core/opl` and `extras/opl` if implicit discovery fails. + +### Domain Expert - Domain Expert + +**Perspective**: Evaluates how well the architecture represents and serves the problem domain and business concepts. + +#### Key Observations +- The domain is split strictly on technical dependency footprint (core vs. extras) rather than by business capability. +- The end-user installation and interaction experience remains explicitly preserved. + +#### Strengths +1. **Preserved User Experience**: Consumers do not need to alter their established `pip install` workflows or import statements. + +#### Concerns +1. **Logical vs Physical Organization** (Impact: Low) + - **Issue**: Grouping by dependency size rather than domain functionality could fragment related modules. + - **Why it matters**: Developers might struggle to find related code if it's split across the boundary. + - **Recommendation**: Clearly document guidelines for where new modules should be placed. + +#### Recommendations +1. **Module Placement Guidelines** (Priority: Medium, Effort: Small) + - **What**: Add clear guidelines in developer documentation. + - **Why**: To help developers decide if a module goes to `core` or `extras` without confusion. + - **How**: Update the repository README or CONTRIBUTING guide. + +### Security Specialist - Security Specialist + +**Perspective**: Reviews the architecture from a security-first perspective, identifying potential vulnerabilities and security implications. + +#### Key Observations +- The architecture cleanly separates heavy dependencies from the core, which acts as a security boundary. + +#### Strengths +1. **Reduced Attack Surface**: Users of the "core" installation won't unnecessarily download and install large third-party packages, minimizing supply chain risks. + +#### Concerns +1. **Meta-package Local References** (Impact: Low) + - **Issue**: The root `setup.py` pulls from local paths. + - **Why it matters**: While standard in monorepos, local file references in packaging need to be tightly controlled in CI environments to prevent path traversal or injection. + - **Recommendation**: Ensure standard, secure git clone practices are used in CI. + +#### Recommendations +1. **Dependency Scanning Boundary** (Priority: Medium, Effort: Small) + - **What**: Ensure dependency scanning tools accurately scan both `core/setup.py` and `extras/setup.py` independently. + - **Why**: To maintain accurate vulnerability reports for both distribution modes. + - **How**: Configure dependabot or equivalent to monitor both setup files. + +### Maintainability Expert - Maintainability Expert + +**Perspective**: Evaluates how well the architecture facilitates long-term maintenance, evolution, and developer understanding. + +#### Key Observations +- Solves a major maintainability headache by eradicating duplicated code. +- Introduces a slight increase in complexity for local development workflows. + +#### Strengths +1. **DRY Principle**: Resolves the worst kind of technical debt by ensuring bug fixes only need to happen in one place. +2. **Clear Constraints**: The physical separation prevents dependency creep into the core modules. + +#### Concerns +1. **Test Discovery Complexity** (Impact: Medium) + - **Issue**: `pytest` historically relies on `__init__.py` files for test discovery. Removing them might require configuration tweaks. + - **Why it matters**: Developers might be confused if running `pytest` locally yields "no tests found". + - **Recommendation**: Verify and document the exact test execution commands required. + +#### Recommendations +1. **Developer Onboarding Docs** (Priority: High, Effort: Small) + - **What**: Update local development documentation. + - **Why**: To explain the split directory structure and new test execution commands to contributors. + - **How**: Add a section in the README detailing the workspace layout. + +### Performance Specialist - Performance Specialist + +**Perspective**: Focuses on performance implications of architectural decisions and suggests optimizations. + +#### Key Observations +- Eliminates the need to parse and load heavy dependencies for core scripts. + +#### Strengths +1. **Consistent Core Execution**: By structurally guaranteeing that core tools don't import heavy dependencies, the cold start time and memory footprint for these CLI tools remains optimal. + +#### Concerns +1. **Namespace Import Overhead** (Impact: Low) + - **Issue**: PEP 420 implicit namespaces involve slightly more disk stat calls during import resolution. + - **Why it matters**: Could fractionally increase startup time. + - **Recommendation**: This is generally negligible in modern Python, but should be noted. + +#### Recommendations +1. **Baseline Startup Time** (Priority: Low, Effort: Small) + - **What**: Measure the startup time of key core scripts before and after the refactor. + - **Why**: To quantitatively ensure no performance regression occurs. + - **How**: Simple `time python -c "import opl.core_module"` checks. + +### Implementation Strategist - Implementation Strategist + +**Perspective**: Evaluates HOW and WHEN changes should be implemented, considering blast radius, reversibility, technical readiness, team capability, and timing. + +#### Key Observations +- The implementation strategy in the ADR is exceptionally well-thought-out, explicitly phasing structure preparation, code migration, and packaging updates. + +#### Strengths +1. **High Reversibility**: If the namespace package approach fails in production, a simple git revert completely undoes the architectural change. +2. **Clear Blast Radius**: The ADR accurately identifies precisely who and what is affected. + +#### Concerns +1. **Merge Conflicts** (Impact: High) + - **Issue**: Moving virtually every file in the `opl/` directory will cause massive merge conflicts for any open feature branches. + - **Why it matters**: It will disrupt other developers and require complex rebasing. + - **Recommendation**: Coordinate the merge tightly with the team. + +#### Recommendations +1. **Merge Coordination** (Priority: High, Effort: Small) + - **What**: Announce a "code freeze" or coordinate the merge timeline. + - **Why**: To minimize disruption to inflight pull requests. + - **How**: Communicate via team channels and merge immediately after a release cycle. + +### AI Engineer - AI Engineer + +**Perspective**: Evaluates the architecture from an AI integration perspective, focusing on practical utility, system evaluation, observability, and agent interaction patterns. + +#### Key Observations +- The split directory structure impacts how AI coding assistants parse and understand the repository context. + +#### Strengths +1. **Clear Context Boundaries**: AI agents will find it easier to understand the strict separation of concerns, reducing the context window needed when working specifically on core tools. + +#### Concerns +1. **Cross-directory Context** (Impact: Low) + - **Issue**: Agents might struggle initially to understand that `core/opl/` and `extras/opl/` form a single logical Python package. + - **Why it matters**: It could lead to incorrect import suggestions by LLMs. + - **Recommendation**: Ensure repository AI instructions explicitly mention the PEP 420 namespace structure. + +#### Recommendations +1. **Update Agent Prompts/Docs** (Priority: Low, Effort: Small) + - **What**: Update AI assistant documentation (e.g., `GEMINI.md`). + - **Why**: To help LLMs navigate the workspace. + - **How**: Add a brief note explaining the PEP 420 namespace layout. + +### Python Expert - Dr. Sarah Chen + +**Perspective**: Advocates for clean, Pythonic solutions following PEP guidelines + +#### Key Observations +- Fully embraces PEP 420 (Implicit Namespace Packages), which is the modern standard for splitting packages across directories. +- Uses standard pip capabilities to preserve installation workflows without relying on hacky `package_dir={'': '..'}` references. + +#### Strengths +1. **Idiomatic Python**: Solves the problem using the exact mechanisms designed by the Python Steering Council for this scenario. + +#### Concerns +1. **`__init__.py` Remnants** (Impact: High) + - **Issue**: If *any* `__init__.py` file is accidentally left in or added to `core/opl/` or `extras/opl/`, Python's import system will treat it as a regular package and break the cross-directory namespace import. + - **Why it matters**: It causes catastrophic import failures. + - **Recommendation**: Add a CI check that strictly enforces the absence of these files in the namespace directories. + +#### Recommendations +1. **Enforce Namespace Package Rules** (Priority: High, Effort: Small) + - **What**: Implement a CI linter rule that explicitly fails if an `__init__.py` file exists in the namespace roots. + - **Why**: To prevent accidental breakage of the architecture. + - **How**: A simple bash script in CI: `[ ! -f "core/opl/__init__.py" ] && [ ! -f "extras/opl/__init__.py" ]`. + +--- + +## Collaborative Discussion + +**Opening Context**: + +**Systems Architect**: "The move to a split-directory namespace package is fundamentally sound and aligns beautifully with modern Python workspace practices. It elegantly solves the duplication issue." + +**Maintainability Expert**: "Agreed completely. Removing duplication is the biggest win here. I am, however, slightly worried about our developer tooling—specifically `pytest` and our linters—dealing gracefully with the lack of `__init__.py` files." + +**Python Expert**: "Modern `pytest` (version 6+) can handle PEP 420 namespaces natively, but we might need to explicitly pass the source directories in our `tox.ini` or GitHub Actions. The much bigger risk is someone accidentally adding an `__init__.py` later, which instantly breaks the implicit namespace across the directories." + +**Implementation Strategist**: "Given the wide-reaching file moves, we need to time this perfectly. Moving every file will cause massive merge conflicts for open PRs. We must coordinate this merge so we don't block the rest of the team." + +### Common Ground + +The team unanimously agrees on: +1. The architectural change is necessary and the PEP 420 approach is the most Pythonic and correct method. +2. Code duplication must be eliminated as a primary objective. +3. The end-user installation commands must remain unchanged. + +### Areas of Debate + +**Topic: Tooling Configuration vs. Architecture** +- **Maintainability Expert**: Argued that fixing tooling is an implementation detail that can be sorted out later. +- **Python Expert**: Countered that because PEP 420 fundamentally changes how modules are resolved, ensuring tooling works is a prerequisite for validating the architecture itself. +- **Resolution**: Tooling configuration and anti-regression checks (like preventing `__init__.py` creation) must be considered critical implementation details that are required before the ADR is marked as fully "implemented". + +### Priorities Established + +**Critical (Address Immediately)**: +1. Validate CI tooling (`pytest`, `flake8`, `mypy`) works seamlessly with PEP 420 namespaces. +2. Coordinate the merge timeline to avoid inflight PR conflicts. + +**Important (Address Soon)**: +1. Add a CI check to prevent `__init__.py` creation in the namespace directories. +2. Update developer onboarding documentation to reflect the new structure. + +**Nice-to-Have (Consider Later)**: +1. Update `GEMINI.md` to inform AI coding assistants about the namespace package structure. + +--- + +## Consolidated Findings + +### Strengths + +1. **Single Source of Truth**: Completely eradicates the technical debt of maintaining duplicated code, ensuring fixes apply globally. +2. **Hard Physical Boundaries**: Physically prevents the leakage of heavy dependencies into the lightweight core package. +3. **Preserved User Experience**: End users see absolutely no change to their installation scripts or import statements. +4. **Idiomatic Solution**: Employs standard Python PEPs rather than brittle packaging hacks. + +### Areas for Improvement + +1. **Tooling Configuration**: + - **Current state**: Tooling assumes standard packages with `__init__.py`. + - **Desired state**: Tooling natively understands the workspace and PEP 420 namespaces. + - **Gap**: Missing validation and potentially configuration tweaks. + - **Priority**: High + - **Impact**: Ensures CI remains reliable. + +2. **Contributor Documentation**: + - **Current state**: No documentation on the split layout. + - **Desired state**: Clear guidance on module placement. + - **Gap**: Missing README updates. + - **Priority**: Medium + - **Impact**: Smooth developer onboarding. + +### Technical Debt + +**High Priority**: +- **Duplicated Code**: + - **Impact**: High maintenance burden and drift risk. + - **Resolution**: Implement this ADR. + - **Effort**: Medium + - **Recommended Timeline**: Immediate + +### Risks + +**Technical Risks**: +- **Namespace Breakage** (Likelihood: Low, Impact: High) + - **Description**: A developer mistakenly adds `__init__.py` to the namespace roots, breaking cross-directory imports. + - **Mitigation**: Add a strict CI check to prevent this. + - **Owner**: Maintainability Expert + +**Operational Risks**: +- **Merge Conflicts** (Likelihood: High, Impact: Medium) + - **Description**: Moving all files disrupts open PRs. + - **Mitigation**: Coordinate the merge tightly with the team and announce a brief code freeze if necessary. + - **Owner**: Implementation Strategist + +--- + +## Recommendations + +### Immediate (0-2 weeks) + +1. **Technical Spike for CI Validation** + - **Why**: Validate that test discovery and linting work seamlessly before merging. + - **How**: Run the existing test suite locally with the new directory structure. + - **Owner**: ADR Author + - **Success Criteria**: All tests pass without modifying the tests themselves. + - **Estimated Effort**: Small + +2. **Merge Coordination** + - **Why**: Prevent painful merge conflicts for other developers. + - **How**: Announce a specific time for the merge and request the team to rebase/merge open work. + - **Owner**: ADR Author / Tech Lead + - **Success Criteria**: PR merged with minimal disruption. + - **Estimated Effort**: Small + +### Short-term (2-8 weeks) + +1. **Implement Anti-Regression CI Check** + - **Why**: Prevent accidental breakage of PEP 420 namespaces. + - **How**: Add a simple step in CI: `if [ -f "core/opl/__init__.py" ] || [ -f "extras/opl/__init__.py" ]; then exit 1; fi`. + - **Owner**: Platform / CI Team + - **Success Criteria**: CI fails if an `__init__.py` is added inappropriately. + - **Estimated Effort**: Small + +2. **Update Developer Docs** + - **Why**: Guide future contributions and explain the new layout. + - **How**: Update the README with the new structure and guidelines for module placement. + - **Owner**: Maintainability Expert + - **Success Criteria**: Documentation clearly explains the difference between `core` and `extras`. + - **Estimated Effort**: Small + +--- + +## Success Metrics + +Define measurable criteria to track improvement: + +1. **Duplicated Code Lines**: + - **Current**: Significant duplication between `opl/` and `core/opl/`. + - **Target**: 0 lines of duplicated code. + - **Timeline**: Immediate (upon merge). + - **Measurement**: Static analysis of the repository. + +2. **CI Pipeline Success**: + - **Current**: Passing. + - **Target**: 100% pass rate on the new directory structure. + - **Timeline**: Immediate. + - **Measurement**: GitHub Actions / CI status. + +--- + +## Follow-up + +**Next Review**: After the implementation PR is opened and validated. + +**Tracking**: Tracked directly via the Pull Request implementing the directory split. + +**Accountability**: +- The ADR author is responsible for implementing the structure and coordinating the merge. +- The CI team is responsible for ensuring the anti-regression checks are in place. + +--- + +## Related Documentation + +**Architectural Decision Records**: +- [ADR-001-reorganize-repository-structure-for-core-and-full-installations.md](../decisions/adrs/ADR-001-reorganize-repository-structure-for-core-and-full-installations.md) - The target of this review. + +--- + +## Appendix + +### Review Methodology + +This review was conducted using the AI Software Architect framework with the following team members: + +- **Systems Architect**: Overall system coherence and patterns +- **Domain Expert**: Business domain representation +- **Security Specialist**: Security analysis and threat modeling +- **Maintainability Expert**: Code quality and technical debt +- **Performance Specialist**: Performance and scalability +- **Implementation Strategist**: Implementation timing, blast radius, and reversibility +- **AI Engineer**: AI system observability and agent interaction patterns +- **Python Expert**: Python best practices and packaging + +Each member reviewed independently, then collaborated to synthesize findings and prioritize recommendations. + +**Pragmatic Mode**: Disabled +- The Pragmatic Enforcer perspective was excluded as per the requested configuration. + +--- + +**Review Complete** \ No newline at end of file diff --git a/.architecture/reviews/comprehensive-review-2026-03-15.md b/.architecture/reviews/comprehensive-review-2026-03-15.md new file mode 100644 index 00000000..a3365e62 --- /dev/null +++ b/.architecture/reviews/comprehensive-review-2026-03-15.md @@ -0,0 +1,318 @@ +# Architecture Review: OPL Repository (Comprehensive) + +**Date**: 2026-03-15 +**Review Type**: Repository +**Reviewers**: Systems Architect, Domain Expert, Security Specialist, Maintainability Expert, Performance Specialist, Implementation Strategist, AI Engineer, Pragmatic Enforcer, Python Expert + +## Executive Summary + +The OPL (Our Performance Library) repository serves as a critical toolkit for the Red Hat Cloud Performance team. It provides a vast array of utilities for cluster data collection, load generation, performance testing, and results parsing. The repository is highly functional and actively maintained, as evidenced by recent commits adding new statistical metrics (percentile95) and fixing label creation. + +However, the architecture reflects an organic growth pattern typical of utility scripts evolving into a larger library. While the core `opl/` package exists, the root directory is cluttered with loose scripts, transient data files, and duplicated directories (`core/` vs `opl/`). This fragmentation hinders discoverability, increases onboarding friction, and risks architectural drift. + +**Overall Assessment**: Adequate (with significant room for structural improvement) + +**Key Findings**: +- **Fragmented Project Structure**: The repository lacks a clear boundary between the core reusable library, CLI entry points, and transient testing/research scripts. The root directory is heavily polluted. +- **Strong Utility Value but Low Cohesion**: The individual utilities (e.g., `cluster_read.py`, `status_data.py`, `generators/`) are powerful and well-utilized, but they operate somewhat independently with varying degrees of shared infrastructure. +- **Complex Dependency Surface**: The `setup.py` lists a wide variety of dependencies (boto3, kafka, psycopg2, locust, jinja2) reflecting the broad scope of the toolset, which could make installation heavy for users who only need a subset of features. + +**Critical Actions**: +- **Consolidate Project Layout**: Implement a standard Python project layout (e.g., `src/opl`) and aggressively move or archive root-level scripts and data files. +- **Clarify Core vs. Scripts**: Clearly delineate library code intended for import from CLI scripts intended for execution. + +--- + +## System Overview + +- **Target**: Entire OPL repository +- **Scope**: Comprehensive architectural review of project structure, code organization, and technical debt. +- **Key Technologies**: Python 3.6+, Pytest, Jinja2, Locust, Kafka, AWS SDK. +- **Architecture Style**: Utility Library / Monolithic CLI Toolset +- **Constraints**: Must maintain backwards compatibility for existing CI/CD pipelines and performance testing workflows that rely on the CLI entry points defined in `setup.py`. + +--- + +## Individual Member Reviews + +### Systems Architect - Systems Architect + +**Perspective**: Focuses on how components work together as a cohesive system and analyzes big-picture architectural concerns. + +#### Key Observations +- The system is fundamentally a collection of CLI tools packaged together, heavily driven by the `console_scripts` entry points in `setup.py`. +- There appears to be an aborted or partial attempt to extract a "core" library, evidenced by the `core/` directory containing its own `setup.py` and `venv`. + +#### Strengths +1. **Clear Execution Model**: The reliance on `console_scripts` entry points provides a standard way for users to discover and run the various tools. +2. **Centralized Configuration**: `setup.py` effectively tracks all dependencies and entry points in one place. + +#### Concerns +1. **Structural Ambiguity** (Impact: High) + - **Issue**: The presence of both `opl/` and `core/opl/` creates confusion about where the source of truth lies. Recent commits ("Propagate forgotten changes to core") highlight the danger of this split. + - **Why it matters**: It leads to duplicated effort, fragmented history, and potential for bugs when changes are not synchronized. + - **Recommendation**: Resolve the `core/` vs root repository dichotomy. Either fully extract `core` into a separate repository/submodule or merge it back completely. + +#### Recommendations +1. **Resolve Core Repository Split** (Priority: High, Effort: Medium) + - **What**: Decide on a single source of truth for the codebase. + - **Why**: Eliminates synchronization overhead and confusion. + - **How**: Propose an ADR to merge `core/` back into the main tree or formalize its separation. + +--- + +### Python Expert - Dr. Sarah Chen + +**Perspective**: Advocates for clean, Pythonic solutions following PEP guidelines. + +#### Key Observations +- The repository contains ~50 Python files in the core package and ~200 functions, indicating a moderately sized codebase. +- The root directory contains numerous unmanaged Python scripts (`aaa.py`, `bbb.py`, `test.py`, `test2.py`, etc.). + +#### Strengths +1. **Standard Packaging**: Utilizes `setuptools` which is the established standard for Python packaging. +2. **Linting Configuration**: Presence of `.flake8` indicates an intent to maintain code style standards. + +#### Concerns +1. **PEP 8 and Project Layout Violations** (Impact: High) + - **Issue**: The root directory is extremely cluttered with `.py` files and `.json` data dumps. + - **Why it matters**: It violates standard Python project layouts, makes the repository intimidating to new contributors, and risks accidental commits of sensitive or transient data. + - **Recommendation**: Adopt a `src/` layout (e.g., `src/opl/`) and move all "playground" scripts to a dedicated, explicitly ignored directory. + +#### Recommendations +1. **Implement Standard Python Layout** (Priority: High, Effort: Small) + - **What**: Reorganize the repository to separate source code, tests, and transient scripts. + - **Why**: Improves discoverability and aligns with Python ecosystem expectations. + - **How**: Create `scripts/` or `playground/` for `aaa.py` et al. Add them to `.gitignore`. + +--- + +### Maintainability Expert - Maintainability Expert + +**Perspective**: Evaluates how well the architecture facilitates long-term maintenance, evolution, and developer understanding. + +#### Key Observations +- Untracked files (`git status` output) show a large number of temporary generator files (`tmp*`) and JSON dumps accumulating in the working directory. +- The `tests/` directory contains 15 files, suggesting reasonable but potentially incomplete coverage for a 50-file library. + +#### Strengths +1. **Automated Testing**: The presence of a `tests/` directory and `.pytest_cache` indicates an active testing culture. + +#### Concerns +1. **Transient File Pollution** (Impact: Medium) + - **Issue**: Data generators and tests appear to be leaving temporary files (`tmp*`) in the source tree instead of using the OS temp directory or a dedicated `build/` or `out/` folder. + - **Why it matters**: Clutters the git status, risks accidental commits, and makes local development messy. + - **Recommendation**: Update file generation logic to use Python's `tempfile` module or write to an explicitly git-ignored output directory. + +#### Recommendations +1. **Clean Up Temporary File Handling** (Priority: Medium, Effort: Small) + - **What**: Ensure all generators and tests clean up after themselves or write to `/tmp`. + - **Why**: Keeps the working directory clean. + - **How**: Refactor file I/O in `opl/generators/` to use `tempfile.TemporaryDirectory`. + +--- + +### Performance Specialist - Performance Specialist + +**Perspective**: Focuses on performance implications of architectural decisions and suggests optimizations. + +#### Key Observations +- The library is designed to handle performance data, including parsing large logs and generating significant load. +- Dependencies include `numpy` and `locust`, which are appropriate choices for performance-focused tooling. + +#### Strengths +1. **Appropriate Tooling**: Leveraging `numpy` for data stats (like the recently added percentile95) is exactly the right architectural choice for performance data processing. + +#### Concerns +1. **Monolithic Dependency Tree** (Impact: Low) + - **Issue**: `setup.py` requires all dependencies (boto3, psycopg2, locust) even if a user only wants to use a specific offline script (like `junit_cli.py`). + - **Why it matters**: Increases installation time and surface area for conflicts. + - **Recommendation**: Consider using `extras_require` for heavy dependencies (e.g., `pip install opl[aws]`). + +#### Recommendations +1. **Modularize Dependencies** (Priority: Low, Effort: Medium) + - **What**: Move domain-specific heavy dependencies to optional extras. + - **Why**: Speeds up installation for CI jobs that only need a subset of the tools. + - **How**: Update `setup.py` `install_requires` and `extras_require`. + +--- + +### Pragmatic Enforcer - Pragmatic Enforcer + +**Perspective**: Rigorously questions whether proposed solutions, abstractions, and features are actually needed right now, pushing for the simplest approach. + +#### Key Observations +- The repository seems to have grown organically by adding scripts as needed, which is very pragmatic. +- However, the attempted split into a `core/` repository seems like an architectural overreach that wasn't fully committed to. + +#### Strengths +1. **Highly Functional**: The tools clearly solve immediate problems for the performance team without unnecessary abstract layers. + +#### Concerns +1. **Over-engineered Repository Split** (Impact: Medium) + - **Issue**: The `core/` directory attempts to create a boundary that the team is failing to maintain (requiring manual sync commits). + - **Why it matters**: It adds overhead without providing clear value. + - **Recommendation**: YAGNI - You aren't going to need a separate core repository if you can't maintain it. Merge it back and rely on Python module namespaces (`opl.core` if necessary) within a single repository. + +#### Recommendations +1. **Eliminate `core/` Directory** (Priority: High, Effort: Small) + - **What**: Remove the separate `core/` setup and integrate its unique changes into the main `opl/` package. + - **Why**: Simplifies the mental model of the repository. + +--- + +## Collaborative Discussion + +**[Systems Architect]**: "The most glaring issue is the structural fragmentation. We have scripts in the root, we have a `core/` directory duplicating effort, and we have the actual `opl/` package. We need a single source of truth." + +**[Python Expert]**: "I agree entirely. A standard Python layout would solve half of our discoverability problems immediately. The current state violates basic PEP 8 project structure conventions." + +**[Pragmatic Enforcer]**: "The `core/` directory is a prime example of failed complexity. We tried to extract a core, found it too hard to maintain across boundaries, and now we are manually syncing changes. We should just merge it back. It's the simplest thing that works." + +**[Maintainability Expert]**: "While we are moving things, we absolutely must address the temporary file spillage. `git status` should be clean after a test run. The generators dropping `tmp*` files into the source tree is unacceptable technical debt." + +**[Performance Specialist]**: "From a performance and usage perspective, the monolithic `setup.py` is okay for now, but as the tool grows, downloading `psycopg2` just to run a local log parser is wasteful. We should keep modular dependencies in mind as a nice-to-have." + +### Common Ground +The team unanimously agrees that repository structure cleanup is the highest priority. The current state is organic but messy, leading to confusion and potential errors (like missed syncs to `core/`). + +### Priorities Established + +**Critical (Address Immediately)**: +1. Resolve the `core/` vs root repository dichotomy (merge back or officially split). +2. Clean up the root directory (move scripts, ignore data files). + +**Important (Address Soon)**: +1. Fix generator logic to use proper temporary directories to stop polluting the working tree. + +**Nice-to-Have (Consider Later)**: +1. Refactor `setup.py` to use `extras_require` for domain-specific dependencies (AWS, DB). + +--- + +## Consolidated Findings + +### Strengths + +1. **Functional Utility Set**: The repository contains highly useful, battle-tested scripts for performance engineering. +2. **Clear CLI Entry Points**: `setup.py` `console_scripts` provide a unified interface for a diverse set of tools. +3. **Appropriate Library Choices**: Use of `numpy` for stats and `locust` for load generation shows good architectural alignment with the domain. + +### Areas for Improvement + +1. **Repository Layout**: + - **Current state**: Cluttered root directory, duplicate `core/` structure. + - **Desired state**: Clean root, standard `src/` layout, single source of truth. + - **Gap**: Missing structural discipline. + - **Priority**: High + - **Impact**: Reduces cognitive load for new developers and prevents sync errors. + +2. **Temporary File Management**: + - **Current state**: Generators and tests leave `tmp*` and `.json` files in the source tree. + - **Desired state**: Working directory remains clean after execution. + - **Gap**: Missing use of standard Python `tempfile` handling. + - **Priority**: Medium + - **Impact**: Improves developer experience and prevents accidental data commits. + +### Technical Debt + +**High Priority**: +- **Duplicated `core/` Logic**: + - **Impact**: Causes merge conflicts, requires manual synchronization, risks bugs. + - **Resolution**: Merge `core/` back into main `opl/` or formally decouple it. + - **Effort**: Small + - **Recommended Timeline**: Immediately. + +**Medium Priority**: +- **Root Script Clutter**: + - **Impact**: Makes the codebase hard to navigate. + - **Resolution**: Move to a `scripts/` directory. + - **Effort**: Small + - **Recommended Timeline**: Next 2 weeks. + +--- + +## Recommendations + +### Immediate (0-2 weeks) + +1. **Create ADR to Resolve `core/` Directory** + - **Why**: Establish a clear decision on the repository's single source of truth. + - **How**: Use the architecture framework to draft an ADR proposing the removal of the duplicated `core/` structure in favor of a unified `opl` package. + - **Owner**: Systems Architect + - **Success Criteria**: ADR merged and accepted by the team. + - **Estimated Effort**: 1 hour. + +2. **Root Directory Cleanup** + - **Why**: Improve discoverability and adhere to Python standards. + - **How**: Create a `playground/` or `scripts/` directory. Move `aaa.py`, `bbb.py`, etc. Update `.gitignore` to explicitly ignore transient `.json` dumps and the playground directory. + - **Owner**: Any developer + - **Success Criteria**: Root directory contains only configuration files (`setup.py`, `README.md`, etc.) and the main package folder. + - **Estimated Effort**: 2 hours. + +### Short-term (2-8 weeks) + +1. **Fix Temporary File Handling** + - **Why**: Stop polluting the git working tree. + - **How**: Audit `opl/generators/` and `tests/` to replace hardcoded temporary file creation with Python's `tempfile` module. + - **Owner**: Any developer + - **Success Criteria**: Running the full test suite and standard generators leaves `git status` clean. + - **Estimated Effort**: 1-2 days. + +### Long-term (2-6 months) + +1. **Modularize Dependencies via `extras_require`** + - **Why**: Reduce installation payload for users who only need specific subsets of the OPL toolkit. + - **How**: Refactor `setup.py` to group dependencies (e.g., `opl[aws]`, `opl[db]`, `opl[load]`). + - **Owner**: Systems Architect / Python Expert + - **Success Criteria**: Base installation `pip install .` installs minimal dependencies required for core functionality. + - **Estimated Effort**: 3 days (requires auditing imports). + +--- + +## Success Metrics + +1. **Repository Cleanliness**: + - **Current**: >20 unmanaged files/scripts in root. + - **Target**: 0 unmanaged scripts in root. + - **Timeline**: 2 weeks. + - **Measurement**: `ls *.py` in root returns only `setup.py`. + +2. **Git Status Purity**: + - **Current**: Many untracked `tmp*` files after execution. + - **Target**: 0 untracked files generated during standard test/run. + - **Timeline**: 1 month. + - **Measurement**: `git status` is clean after running `pytest`. + +--- + +## Follow-up + +**Next Review**: 2026-06-15 + +**Tracking**: +- Create GitHub issues for immediate actions (Core resolution, Root cleanup). + +**Recalibration**: +After implementing the root cleanup: +``` +"Start architecture recalibration for OPL project structure" +``` + +--- + +## Appendix + +### Review Methodology + +This review was conducted using the AI Software Architect framework with the following team members: +- **Systems Architect**: Overall system coherence and patterns +- **Python Expert**: Language specific best practices and packaging +- **Maintainability Expert**: Code quality and technical debt +- **Performance Specialist**: Performance and scalability +- **Pragmatic Enforcer**: Challenging complexity and YAGNI + +**Pragmatic Mode**: Balanced +- All recommendations evaluated through YAGNI lens, specifically regarding the over-complicated `core/` repository split. + +**Review Complete** \ No newline at end of file diff --git a/.architecture/reviews/initial-system-analysis.md b/.architecture/reviews/initial-system-analysis.md new file mode 100644 index 00000000..3856b3a0 --- /dev/null +++ b/.architecture/reviews/initial-system-analysis.md @@ -0,0 +1,169 @@ +# Initial System Analysis + +**Project**: OPL (Our Performance Library) +**Date**: 2026-03-15 +**Analysis Type**: Initial Setup Assessment +**Analysts**: Systems Architect, Python Expert, Performance Specialist, Security Specialist, Maintainability Expert + +--- + +## Executive Summary + +OPL (Our Performance Library) is a comprehensive Python-based performance testing and data processing utility suite. It serves as a central repository for performance-related scripts, data generators, and analysis tools used by the Red Hat Cloud Performance team. The project handles a wide variety of tasks including cluster data collection (OpenShift/Kubernetes), result generation for various services (Inventory, Insights), database management, and integration with performance tracking tools like Horreum. + +The system is a collection of utilities rather than a single monolithic application, characterized by a large number of entry points defined in `setup.py`. It leverages a diverse stack including Jinja2 for templating, Boto3 for AWS interaction, Kafka for messaging, and various data processing libraries like NumPy and DeepDiff. The codebase shows significant history with many specialized scripts and a testing suite that covers core components. + +**Overall Assessment**: Adequate + +**Key Findings**: +- **Extensive Utility Set**: High coverage of performance engineering needs, from data generation to reporting. +- **Fragmented Structure**: The root directory is cluttered with various JSON, YAML, and Python scripts, making it difficult to distinguish between core library code and transient data/test scripts. +- **Complex Dependency Profile**: Large list of dependencies reflecting its role as an integration hub for multiple performance testing workflows. + +**Critical Recommendations**: +- **Root Directory Cleanup**: Move loose scripts and data files into appropriate subdirectories to improve project discoverability. +- **Formalize API Boundaries**: Better define the core library (`opl/`) vs. the CLI entry points to prevent circular dependencies and improve reuse. + +--- + +## System Overview + +### Project Information + +**Primary Language(s)**: Python + +**Frameworks**: Jinja2, Locust, Pytest + +**Architecture Style**: Distributed Utility Library / CLI Toolset + +**Deployment**: Python Package (PyPI/Internal), Containerized (Containerfile present) + +**Team Size**: Not explicitly defined, but maintained by Red Hat Performance Team. + +**Project Age**: Several years (active development since at least 2021 based on test data). + +### Technology Stack + +**Backend**: +- Python 3.6+ +- Jinja2 (Templating) +- Boto3 (AWS SDK) +- Kafka-python (Messaging) +- Psycopg2 (PostgreSQL) + +**Data Processing**: +- NumPy +- DeepDiff +- Tabulate + +**Testing**: +- Pytest +- Black/Flake8 (Linting) +- Pyfakefs + +**Infrastructure**: +- Docker (Containerfile) +- Tekton (CI/CD integration files) +- Horreum (Performance result tracking) + +### Project Structure + +``` +/ +├── opl/ # Core library code +│ ├── generators/ # Data generation logic +│ ├── investigator/ # Investigation/Analysis tools +│ └── ... # Various utility modules +├── core/ # Potential core/subset of the library +├── tests/ # Test suite +├── aaa/ # Large collection of JSON test/log data +├── DELME/ # Transient measurement data +├── .architecture/ # Architecture framework +└── ... # Loose scripts (aaa.py, bbb.py, test.py, etc.) +``` + +--- + +## Individual Member Analyses + +### Systems Architect - Systems Architect + +**Perspective**: Overall system coherence and architectural patterns. + +#### Current State Assessment +The system operates more as a "toolbox" than a unified application. While `opl/` is the core package, the high number of console scripts indicates that it is primarily consumed as a set of CLI tools. The presence of a `core/` directory suggests an attempt at tiered layering, but the relationship between root `opl/` and `core/` needs clarification. + +#### Strengths Identified +1. **Clear Entry Points**: `setup.py` defines a comprehensive list of CLI tools, making the library's capabilities discoverable for power users. +2. **Modular Generators**: Data generation is well-encapsulated in `opl/generators/`. + +#### Concerns Raised +1. **Lack of Encapsulation** (Impact: Medium) + - **Issue**: Many scripts are in the root directory rather than being integrated into the `opl/` package. + - **Recommendation**: Integrate useful root scripts into `opl/` and delete or move "playground" scripts (test.py, etc.). + +### Python Expert - Dr. Sarah Chen + +**Perspective**: Pythonic best practices and language-specific patterns. + +#### Current State Assessment +The project uses modern Python features and has a solid `setup.py`. However, the proliferation of `.py` files in the root directory (test.py, test2.py, test3.py, aaa.py) violates PEP 8 and standard project structures. + +#### Strengths Identified +1. **Standard Packaging**: Uses `setuptools` correctly with `entry_points`. +2. **Type Hints**: Some evidence of type hint usage in imports. + +#### Concerns Raised +1. **Root Directory Clutter** (Impact: High) + - **Issue**: Non-package Python files in root. + - **Why It Matters**: Confuses module resolution and makes the project look unmaintained. + - **Recommendation**: Use a `src/` layout or move all scripts to `scripts/` or `opl/`. + +### Performance Specialist - Performance Specialist + +**Perspective**: Performance engineering and scalability of the tools themselves. + +#### Current State Assessment +The toolset is designed for performance engineers. It uses efficient libraries like NumPy for data processing. + +#### Strengths Identified +1. **Locust Integration**: Built-in support for load testing. +2. **Efficient Data Handling**: Uses generators and streaming patterns for large log files. + +#### Concerns Raised +1. **Database Bottlenecks** (Impact: Medium) + - **Issue**: Direct `psycopg2` usage might lead to unpooled connections in high-concurrency CLI usage. + - **Recommendation**: Ensure connection pooling is used if any scripts are long-running or concurrent. + +--- + +## Architectural Health Assessment + +### Code Quality +**Rating**: 6/10 +**Observations**: Core library code seems reasonable, but the surrounding environment is messy. + +### Testing +**Rating**: 7/10 +**Observations**: Good variety of tests in `tests/`, including generators and CLI utilities. Use of `pyfakefs` shows sophisticated testing of filesystem-interacting code. + +### Documentation +**Rating**: 5/10 +**Observations**: `README.md` exists but is relatively brief. Architectural intent is mostly implicit. + +--- + +## Recommendations + +### Immediate Actions (0-2 Weeks) +1. **Archive Loose Files**: Move `aaa.py`, `bbb.py`, `test.py` etc. to a `research/` or `playground/` folder if they are still needed, otherwise delete. +2. **Document `core/` vs `opl/`**: Create an ADR explaining the relationship between these two components. + +### Short-Term Actions (2-8 Weeks) +1. **Standardize Logging**: Ensure all CLI tools use a unified logging configuration from `opl/`. +2. **Expand Test Coverage**: Focus on `opl/investigator` which seems less covered by current tests. + +--- + +**Analysis Complete** +**Next Review**: 2026-06-15 diff --git a/.architecture/templates/AGENTS.md b/.architecture/templates/AGENTS.md new file mode 100644 index 00000000..f82de667 --- /dev/null +++ b/.architecture/templates/AGENTS.md @@ -0,0 +1,244 @@ +# AGENTS.md - AI Software Architect Framework + + + +> **For [AI Assistant] Users**: This file contains cross-platform instructions for the AI Software Architect framework. For assistant-specific features and detailed procedures, see [.architecture/agent_docs/README.md](.architecture/agent_docs/README.md). + +## Project Overview + +[PROJECT_NAME] uses the AI Software Architect framework to implement rigorous software architecture practices with AI assistant collaboration. + +**Technology Stack**: [PROJECT_TECH_STACK] + +The framework provides: +- **Architecture Documentation**: Centralized repository of architectural decisions and reviews +- **Multi-Perspective Reviews**: Specialized reviewers from different architectural domains +- **Decision Records (ADRs)**: Structured documentation of architectural decisions +- **Recalibration Process**: Translating architectural reviews into implementation plans +- **Pragmatic Mode**: Optional YAGNI enforcement to prevent over-engineering +- **Progressive Disclosure**: Detailed procedures in `.architecture/agent_docs/` directory + +## Framework Setup + +### Directory Structure + +``` +.architecture/ +├── members.yml # Architecture review team members +├── principles.md # Core architectural principles +├── config.yml # Framework configuration (including pragmatic mode) +├── decisions/ # Architectural Decision Records (ADRs) +│ └── adrs/ # Numbered ADR documents +├── reviews/ # Architecture review documents +├── recalibration/ # Implementation planning documents +└── templates/ # Templates for ADRs, reviews, etc. + +.architecture/agent_docs/ # Detailed AI assistant procedures +├── README.md # Navigation guide +├── workflows.md # Step-by-step procedures +└── reference.md # Advanced topics and troubleshooting +``` + +**👉 For detailed setup procedures, see [.architecture/agent_docs/workflows.md § Setup Procedures](.architecture/agent_docs/workflows.md#setup-procedures)** + +## Core Workflows + +**👉 For detailed workflow procedures, see [.architecture/agent_docs/workflows.md](.architecture/agent_docs/workflows.md)** + +### Requesting Architecture Reviews + +Architecture reviews provide multi-perspective analysis of architectural decisions. + +**Command patterns:** +- "Start architecture review for version X.Y.Z" +- "Start architecture review for [feature name]" +- "Ask [Specialist] to review [component]" + +Review documents created in `.architecture/reviews/`. + +**See**: [.architecture/agent_docs/workflows.md § Architecture Review Workflows](.architecture/agent_docs/workflows.md#architecture-review-workflows) + +### Creating Architectural Decision Records (ADRs) + +ADRs document significant architectural decisions. + +**Command patterns:** +- "Create ADR for [topic]" +- "Document architectural decision for [topic]" + +ADRs stored in `.architecture/decisions/adrs/` with sequential numbering. + +**See**: [.architecture/agent_docs/workflows.md § ADR Creation Workflow](.architecture/agent_docs/workflows.md#adr-creation-workflow) + +### Enabling Pragmatic Mode + +Pragmatic mode adds YAGNI enforcement to prevent over-engineering. + +**Command patterns:** + +- "Enable pragmatic mode" +- "Turn on YAGNI enforcement" + +Configuration in `.architecture/config.yml` with three intensity modes: Strict, Balanced, Lenient. + +**See**: [.architecture/agent_docs/reference.md § Pragmatic Guard Mode](.architecture/agent_docs/reference.md#pragmatic-guard-mode) + +### Architectural Recalibration + +Translate architecture review findings into actionable implementation plans. + +**Command patterns:** +- "Start architecture recalibration for version X.Y.Z" +- "Recalibrate architecture for [feature name]" + +Recalibration documents stored in `.architecture/recalibration/`. + +**See**: [.architecture/agent_docs/reference.md § Architecture Recalibration](.architecture/agent_docs/reference.md#architecture-recalibration) + +## Architecture Principles + +This project follows architectural principles defined in `.architecture/principles.md`. Key principles include: + +- **Livable Code**: Design for developers who inhabit the codebase +- **Clarity over Cleverness**: Prefer simple, clear designs +- **Separation of Concerns**: Clear boundaries and single responsibilities +- **Evolvability**: Facilitate change without complete rewrites +- **Observability**: System provides insights into behavior and state +- **Security by Design**: Security integral to architecture, not afterthought +- **Domain-Centric Design**: Reflect and serve the problem domain +- **Pragmatic Simplicity**: Value working solutions over theoretical perfection + +For detailed explanations and application guidelines, see `.architecture/principles.md`. + +## Build & Test (Framework Development) + +If working on the AI Software Architect framework itself: + +### Testing Framework Components + +```bash +# Verify directory structure +ls -la .architecture/ + +# Check configuration +cat .architecture/config.yml + +# List architecture members +cat .architecture/members.yml + +# View templates +ls .architecture/templates/ +``` + +### Validating ADRs and Reviews + +- ADRs should follow template at `.architecture/templates/adr-template.md` +- Reviews should follow template at `.architecture/templates/review-template.md` +- Recalibration should follow template at `.architecture/templates/recalibration_plan.md` + +## Project-Specific Information + +[PROJECT_TECH_STACK] + +[PROJECT_BUILD_COMMANDS] + +[PROJECT_TEST_COMMANDS] + +[PROJECT_CONVENTIONS] + +## Assistant-Specific Features + +The AI Software Architect framework provides enhanced capabilities for specific AI coding assistants: + +### Claude Code + +Claude Code users have access to enhanced features including: + +- **Claude Skills**: Reusable skills for setup, reviews, ADR creation, and status checks +- **MCP Server Integration**: Tools for architecture operations via Model Context Protocol +- **Slash Commands**: Custom commands for framework operations +- **Enhanced Setup**: Intelligent project analysis and template customization + +**See [CLAUDE.md](../CLAUDE.md) for complete Claude Code documentation.** + +### Cursor + +Cursor users can configure the framework via: + +- **Configuration**: See `.coding-assistants/cursor/README.md` +- **Rules**: Custom rules for architecture operations +- **Integration**: Tab completion and inline suggestions + +**See [.coding-assistants/cursor/README.md](../.coding-assistants/cursor/README.md) for details.** + +### GitHub Copilot / Codex + +Copilot users can access framework features via: + +- **Configuration**: See `.coding-assistants/codex/README.md` +- **Comments**: Use comments to trigger architecture operations +- **Integration**: Inline suggestions for ADRs and reviews + +**See [.coding-assistants/codex/README.md](../.coding-assistants/codex/README.md) for details.** + +### Other AI Assistants + +The framework works with any AI assistant that can read markdown files and follow structured instructions. Key entry points: + +- **This file (AGENTS.md)**: Cross-platform instructions +- **.architecture/**: All framework artifacts and templates +- **principles.md**: Architectural principles to apply +- **members.yml**: Available architecture reviewers +- **templates/**: Templates for ADRs, reviews, and recalibration + +## Additional Resources + +### Detailed Documentation +- **Workflow Procedures**: `.architecture/agent_docs/workflows.md` +- **Advanced Topics**: `.architecture/agent_docs/reference.md` +- **Documentation Guide**: `.architecture/agent_docs/README.md` + +### Framework Files +- **Framework Principles**: `.architecture/principles.md` +- **Architecture Members**: `.architecture/members.yml` +- **Configuration**: `.architecture/config.yml` + +### Templates & Examples +- **ADR Template**: `.architecture/templates/adr-template.md` +- **Review Template**: `.architecture/templates/review-template.md` +- **Recalibration Template**: `.architecture/templates/recalibration_plan.md` +- **Example ADRs**: `.architecture/decisions/adrs/` +- **Example Reviews**: `.architecture/reviews/` + +--- + +**Framework Version**: [FRAMEWORK_VERSION] +**Documentation Version**: 2.0.0 (Progressive Disclosure - ADR-006) +**Last Updated**: [LAST_UPDATED] +**Maintained By**: AI Software Architect Framework + + diff --git a/.architecture/templates/adr-template.md b/.architecture/templates/adr-template.md new file mode 100644 index 00000000..13a49d15 --- /dev/null +++ b/.architecture/templates/adr-template.md @@ -0,0 +1,281 @@ +# ADR-XXX: [Title] + +## Status + +[Draft | Proposed | Accepted | Deprecated | Superseded] + +If superseded, link to the new ADR: [New ADR Link] + +## Context + +[Describe the context and problem statement that led to this decision. Include any relevant constraints, requirements, or background information. Reference the architectural review findings if applicable.] + +## Decision Drivers + +* [Driver 1: Briefly describe a factor influencing the decision] +* [Driver 2: ...] +* [Driver n: ...] + +## Decision + +[Describe the decision that was made. Be clear and precise about the architectural change being implemented.] + +**Architectural Components Affected:** +* [Component 1] +* [Component 2] +* [...] + +**Interface Changes:** +* [Detail any changes to public interfaces] + +## Consequences + +### Positive + +* [Positive consequence 1] +* [Positive consequence 2] +* [...] + +### Negative + +* [Negative consequence 1] +* [Negative consequence 2] +* [...] + +### Neutral + +* [Neutral consequence 1] +* [Neutral consequence 2] +* [...] + +## Implementation Strategy + +> **Note**: This section externalizes the "senior thinking" about HOW and WHEN to implement this decision, not just WHAT to implement. + +### Blast Radius + +**Impact Scope**: [Describe the scope of impact if this decision fails or needs reversal] + +**Affected Components**: +- [Component/Service 1] - [Impact description] +- [Component/Service 2] - [Impact description] +- [...] + +**Affected Teams**: +- [Team 1] - [Coordination needs] +- [Team 2] - [Coordination needs] +- [...] + +**User Impact**: [Description of user-facing risks or changes] + +**Risk Mitigation**: +- [Mitigation strategy 1] +- [Mitigation strategy 2] +- [...] + +### Reversibility + +**Reversibility Level**: [High | Medium | Low | Irreversible] + +**Rollback Feasibility**: +- [Description of how easily this can be undone] +- [What makes it reversible or irreversible] +- [Time and effort required for reversal] + +**Migration Paths**: +- **Forward Migration**: [Path to implement this decision] +- **Rollback Migration**: [Path to reverse this decision if needed] +- **Evolution Path**: [How this decision can evolve over time] + +**Options Preserved**: +- [Future options this decision keeps open] +- [Alternatives that remain viable] + +**Commitments Made**: +- [Decisions this locks us into] +- [Future options this decision closes] + +### Sequencing & Timing + +**Prerequisites**: +- [ ] [Prerequisite 1 - with status] +- [ ] [Prerequisite 2 - with status] +- [ ] [...] + +**System Readiness**: +- **Observability**: [Is monitoring/logging adequate for this change?] +- **Dependencies**: [Are dependent systems stable and ready?] +- **Infrastructure**: [Is infrastructure sufficient?] +- **Data Migration**: [Are data migration paths clear and tested?] + +**Team Readiness**: +- **Understanding**: [Does the team understand the patterns being introduced?] +- **Skills**: [Does the team have necessary skills?] +- **Training Needs**: [What training or documentation is required?] +- **Consensus**: [Is there team buy-in for this approach?] + +**Sequencing Concerns**: +- [Should other changes happen first?] +- [What coordination is required?] +- [Are there timing dependencies?] + +**Readiness Assessment**: [Ready to implement | Needs preparation | Requires prerequisites] + +### Social Cost + +**Learning Curve**: [Low | Medium | High] +- [Description of what team members need to learn] +- [Time estimate for team to become proficient] + +**Cognitive Load**: +- [Mental overhead this pattern/approach adds] +- [Complexity vs. clarity trade-off analysis] + +**Clarity Assessment**: +- **Will this help more than confuse?**: [Yes | No | Depends] +- **Explanation required**: [What needs to be documented or explained?] +- **Onboarding impact**: [Effect on new team members] + +**Documentation Needs**: +- [ ] [Documentation item 1] +- [ ] [Documentation item 2] +- [ ] [...] + +### Confidence Assessment + +**Model Correctness Confidence**: [High | Medium | Low] +- [Are we confident in the architectural model, not just the implementation?] +- [What could make tests pass while the model is still wrong?] + +**Assumptions**: +1. [Assumption 1] - **Validation**: [How validated is this?] +2. [Assumption 2] - **Validation**: [How validated is this?] +3. [...] + +**Uncertainty Areas**: +- [Area 1 where we're uncertain] +- [Area 2 where we're uncertain] +- [...] + +**Validation Approach**: +- [How will we validate the model is correct?] +- [What experiments or prototypes are needed?] +- [What feedback loops will verify assumptions?] + +**Edge Cases**: +- [Edge case 1 not captured by testing] +- [Edge case 2 not captured by testing] +- [...] + +## Implementation + +[Provide a high-level implementation plan, including any phasing or migration strategies.] + +**Phase 1: [Phase Name]** +* [Implementation step 1] +* [Implementation step 2] +* [...] + +**Phase 2: [Phase Name]** +* [Implementation step 1] +* [Implementation step 2] +* [...] + +## Alternatives Considered + +### [Alternative 1] + +[Describe alternative approach] + +**Pros:** +* [Pro 1] +* [Pro 2] + +**Cons:** +* [Con 1] +* [Con 2] + +### [Alternative 2] + +[Describe alternative approach] + +**Pros:** +* [Pro 1] +* [Pro 2] + +**Cons:** +* [Con 1] +* [Con 2] + +## Pragmatic Enforcer Analysis + +**Reviewer**: [NAME] +**Mode**: [Strict | Balanced | Lenient] + +**Note**: *This section only appears when pragmatic_mode is enabled in `.architecture/config.yml`* + +**Overall Decision Complexity Assessment**: +[High-level assessment of whether this decision maintains appropriate simplicity or shows signs of over-engineering. Consider: Is this solving a current problem or a future possibility? Are we adding complexity for speculative needs?] + +**Decision Challenge**: + +**Proposed Decision**: "[Briefly quote the decision being made]" + +**Necessity Assessment**: [Score 0-10] +- **Current need**: [Is this decision addressing a current, concrete requirement?] +- **Future need**: [What's the likelihood and timeframe for this being actually needed?] +- **Cost of waiting**: [What breaks if we defer this decision? What's the cost of implementing later?] +- **Evidence of need**: [What concrete evidence justifies this decision now?] + +**Complexity Assessment**: [Score 0-10] +- **Added complexity**: [What complexity does this decision introduce to the system?] +- **Maintenance burden**: [What is the ongoing cost to maintain this?] +- **Learning curve**: [What is the impact on team/new developers?] +- **Dependencies introduced**: [What new dependencies or abstractions are added?] + +**Alternative Analysis**: +[Review of whether simpler alternatives were adequately considered] +- Are the listed alternatives genuinely simpler, or just different? +- Is there a "do nothing" or "minimal change" option missing? +- Could we solve this with existing tools/patterns? + +**Simpler Alternative Proposal**: +[If applicable, concrete proposal for a simpler approach that meets current actual requirements. This might be: +- A phased approach (implement minimal now, extend later) +- Using existing tools instead of custom solutions +- Deferring the decision until more information is available +- A less abstract/more direct solution] + +**Recommendation**: [✅ Approve decision | ⚠️ Approve with simplifications | ⏸️ Defer until triggered | ❌ Recommend against] + +**Justification**: +[Clear reasoning for the recommendation, balancing current needs vs future flexibility vs complexity costs] + +**If Deferring or Simplifying**: +- **Trigger conditions**: [What would trigger implementing this decision or the full version?] +- **Minimal viable alternative**: [What's the simplest thing that could work right now?] +- **Migration path**: [If we implement the minimal version, how do we migrate later if needed?] + +**Pragmatic Score**: +- **Necessity**: [X/10] +- **Complexity**: [X/10] +- **Ratio**: [Complexity/Necessity = X.X] *(Target: <1.5 for balanced mode)* + +**Overall Assessment**: +[Summary judgment: Does this decision represent appropriate engineering for current needs, or potential over-engineering for future possibilities?] + +## Validation + +**Acceptance Criteria:** +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [...] + +**Testing Approach:** +* [Describe how the implementation of this decision will be tested] + +## References + +* [Architectural Review X.Y.Z](link-to-review) +* [Reference 1](link) +* [Reference 2](link) \ No newline at end of file diff --git a/.architecture/templates/config.yml b/.architecture/templates/config.yml new file mode 100644 index 00000000..c0e157aa --- /dev/null +++ b/.architecture/templates/config.yml @@ -0,0 +1,393 @@ +# AI Software Architect Configuration +# This file controls operational modes and behavior of the architectural review framework + +# ============================================================================== +# PRAGMATIC GUARD MODE +# ============================================================================== +# The Pragmatic Guard Mode adds a specialized "Pragmatic Enforcer" architect +# who actively challenges complexity, questions abstractions, and pushes for +# the simplest solutions that meet current requirements. +# +# Enable this mode to guard against over-engineering and ensure YAGNI +# (You Aren't Gonna Need It) principles are applied rigorously. + +pragmatic_mode: + # Enable or disable Pragmatic Guard Mode + # Default: false (opt-in feature) + enabled: false + + # Intensity level controls how aggressively the Pragmatic Enforcer challenges + # complexity and pushes for simplicity + # + # - strict: Challenges aggressively, requires strong justification for any complexity + # Questions every "should" and "could", pushes for absolute minimal implementation + # + # - balanced: Challenges thoughtfully, accepts justified complexity (RECOMMENDED) + # Seeks middle ground between simplicity and best practices + # Questions "should" but accepts reasonable "must" + # + # - lenient: Raises concerns without blocking, suggests simpler alternatives as options + # Focuses on major complexity additions only + # Questions significant departures from simplicity + # + # Default: balanced + intensity: balanced + + # Control which review phases include the Pragmatic Enforcer + apply_to: + # Include in individual architect reviews + individual_reviews: true + + # Include in collaborative discussion phase + collaborative_discussions: true + + # Include when planning implementation steps + implementation_planning: true + + # Include during ADR (Architecture Decision Record) creation + adr_creation: true + + # Include in specific architect reviews (when user asks single architect) + specific_reviews: true + + # Exemptions: Areas where strict best practices should be maintained + # regardless of pragmatic mode intensity. The Pragmatic Enforcer will + # still participate but will not challenge security/compliance requirements. + exemptions: + # Never compromise on security-critical features + # Examples: authentication, authorization, encryption, input validation + security_critical: true + + # Never compromise on data integrity + # Examples: database transactions, data validation, backup strategies + data_integrity: true + + # Never compromise on compliance requirements + # Examples: GDPR, HIPAA, PCI-DSS, audit logging + compliance_required: true + + # Never compromise on accessibility requirements + # Examples: WCAG compliance, screen reader support + accessibility: true + + # Triggers: Situations that activate pragmatic analysis and challenges + # Disable specific triggers if you want to skip analysis for those cases + triggers: + # Challenge when adding new abstraction layers + # Examples: new interfaces, abstract base classes, dependency injection containers + new_abstraction_layer: true + + # Challenge when adding new external dependencies + # Examples: new libraries, frameworks, services + new_dependency: true + + # Challenge when introducing new architectural patterns + # Examples: repository pattern, strategy pattern, observer pattern + new_pattern_introduction: true + + # Challenge when expanding scope beyond initial requirements + # Examples: adding "nice to have" features, building for imagined future needs + scope_expansion: true + + # Challenge performance optimizations without evidence of problems + # Examples: caching layers, database indexes, query optimization + performance_optimization: true + + # Challenge comprehensive test infrastructure upfront + # Examples: full integration tests, E2E tests, property tests before code exists + test_infrastructure: true + + # Challenge adding flexibility or configurability + # Examples: feature flags, plugin systems, configuration frameworks + flexibility_addition: true + + # Thresholds: Numerical values that help determine when to challenge + thresholds: + # Minimum complexity score to trigger a challenge (0-10) + # Complexity is assessed by: abstraction layers, file count, dependencies, LOC + # Default: 5 (challenge moderate to high complexity) + min_complexity_score: 5 + + # Minimum necessity score to accept without challenge (0-10) + # Necessity is assessed by: current requirements, immediate value, cost of deferral + # Default: 7 (must be clearly necessary to skip challenge) + min_necessity_score: 7 + + # Maximum acceptable complexity-to-necessity ratio + # If complexity score / necessity score > threshold, challenge strongly + # Default: 1.5 (complexity should not exceed necessity by more than 50%) + max_complexity_ratio: 1.5 + + # Behavioral configuration + behavior: + # Require explicit justification for complexity + # When true, architects must proactively justify complex solutions + require_justification: true + + # Always propose simpler alternative + # When true, Pragmatic Enforcer always suggests a simpler approach + always_propose_alternative: true + + # Calculate and show cost of waiting + # When true, includes analysis of what happens if implementation is deferred + show_cost_of_waiting: true + + # Track deferred decisions + # When true, maintains a log of decisions deferred for future implementation + track_deferrals: true + + # Deferral log location + deferral_log: .architecture/deferrals.md + + # Question templates: Customize the questions the Pragmatic Enforcer asks + # Leave empty to use defaults + custom_questions: + necessity: [] + # - "Do we need this right now?" + # - "What breaks if we don't implement this?" + + simplicity: [] + # - "What's the simplest thing that could work?" + # - "Can we do this with less code?" + + cost: [] + # - "What's the cost of implementing this now?" + # - "What's the cost of waiting?" + + alternatives: [] + # - "What if we just...?" + # - "Could we use an existing tool?" + + best_practices: [] + # - "Does this best practice apply to our context?" + # - "Is this over-engineering for our scale?" + +# ============================================================================== +# IMPLEMENTATION GUIDANCE +# ============================================================================== +# Configure how AI assistants implement features when you use the command: +# "Implement X as the architects" +# +# This allows you to specify your development methodology, coding influences, +# and practices once, and have them applied consistently across all implementations. +# +# Benefits: +# - 90% reduction in prompt length (no more repeating preferences) +# - Consistent application of best practices +# - Team standards documented in version control +# - Better onboarding (new developers see documented practices) + +implementation: + # Enable or disable implementation guidance + # When enabled, "Implement X as the architects" will apply your configuration + # When disabled, implementations proceed without special guidance + # Default: true + enabled: true + + # Primary development methodology + # Specifies the overall approach to writing code + # Options: TDD, BDD, DDD, Test-Last, Exploratory, or custom + # + # - TDD (Test-Driven Development): Write tests first, red-green-refactor + # - BDD (Behavior-Driven Development): Behavior-focused tests, outside-in + # - DDD (Domain-Driven Design): Domain modeling, bounded contexts + # - Test-Last: Implementation first, tests after + # - Exploratory: Experiment, then codify + # + # Default: TDD + methodology: "TDD" + + # Coding influences - thought leaders whose practices to follow + # List specific books, talks, or articles that guide your approach + # + # Format: "Name - Source (focus area)" or just "Name - Source" + # + # Examples: + # - "Kent Beck - TDD by Example" + # - "Sandi Metz - POODR, 99 Bottles" + # - "Martin Fowler - Refactoring" + # - "Gary Bernhardt - Destroy All Software" + # - "Eric Evans - Domain-Driven Design" + # + # Can be organized as simple list or grouped by focus area + influences: + - "Kent Beck - TDD by Example (methodology fundamentals)" + - "Sandi Metz - POODR, 99 Bottles (OO design and refactoring)" + - "Martin Fowler - Refactoring (patterns and code smells)" + # Add more influences as needed + # - "Your favorite author - Their book/source" + + # Language-specific practices (optional) + # Customize for your project's primary language(s) + # Add sections for each language you use + languages: + # Example: Ruby configuration + # ruby: + # style_guide: "Rubocop" + # idioms: "Prefer blocks over loops, use meaningful method names" + # frameworks: + # rails: "Follow conventions but question them" + # sinatra: "Keep it simple" + + # Example: JavaScript/TypeScript configuration + # javascript: + # style_guide: "ESLint with Airbnb config" + # idioms: "Prefer const, use arrow functions, destructure" + # frameworks: + # react: "Functional components, hooks" + # vue: "Composition API" + + # Example: Python configuration + # python: + # style_guide: "PEP 8 via Black formatter" + # idioms: "Pythonic patterns, list comprehensions" + # frameworks: + # django: "Fat models, thin views" + # flask: "Keep it minimal" + + # Testing approach (optional) + # Specify your testing framework, style, and goals + testing: + # Testing framework to use + # framework: "RSpec" # or Minitest, Jest, pytest, JUnit, etc. + + # Testing style + # style: "Outside-in TDD" # or Inside-out, Detroit school, London school + + # Testing approach + # approach: "Mock judiciously, prefer real objects when practical" + + # Coverage goals + # coverage: "High for business logic, integration tests for critical paths" + + # Test speed targets + # speed: "Unit tests <100ms, integration tests <1s" + + # Refactoring guidelines (optional) + # When and how to refactor code + refactoring: + # When to refactor + # when: + # - "After tests green (red-green-REFACTOR)" + # - "When code smells emerge (long methods, duplication)" + # - "Rule of Three: refactor on third occurrence" + + # Refactoring principles + # principles: + # - "Small methods (≤5 lines per Sandi Metz)" + # - "Clear names over comments" + # - "Simple design over clever code" + + # Quality standards (optional) + # Define what "done" means for your team + quality: + # Definition of done checklist + # definition_of_done: + # - "Tests passing (all green)" + # - "Code refactored for clarity" + # - "No obvious code smells" + # - "Code reviewed by team" + # - "Documentation updated (if public API)" + + # Quality priorities (in order) + # priorities: + # - "Clarity first (code is read more than written)" + # - "Simplicity second (avoid premature abstraction)" + # - "Performance third (optimize when measured need exists)" + + # Security practices (optional but recommended) + # These are always applied, even with pragmatic_mode enabled + # Security practices are exempt from YAGNI challenges + security: + # Mandatory security practices + # Always applied regardless of other settings + # mandatory_practices: + # - "Input validation (whitelist approach)" + # - "Output encoding (context-aware)" + # - "Parameterized queries (no string concatenation)" + # - "Authentication checks (verify identity)" + # - "Authorization checks (verify permissions)" + + # Security influences + # influences: + # - "OWASP Secure Coding Practices" + # - "SEI CERT Coding Standards" + + # Performance considerations (optional) + # Configure performance-aware implementation + performance: + # Is this a performance-critical system? + # If true, adds performance-focused practices and influences + # critical: false + + # Performance practices + # practices: + # - "Profile before optimizing" + # - "Benchmark critical paths" + # - "Set performance budgets" + + # Performance influences (when critical: true) + # influences: + # - "Brendan Gregg - Systems Performance" + +# ============================================================================== +# GENERAL CONFIGURATION +# ============================================================================== + +# Review process configuration +review_process: + # Require all members to review before collaborative phase + require_all_members: true + + # Maximum time for individual review phase (for planning purposes) + max_individual_phase_days: 3 + + # Maximum time for collaborative discussion (for planning purposes) + max_collaborative_phase_days: 2 + +# Architecture Decision Record (ADR) configuration +adr: + # Numbering format: sequential | date-based + numbering_format: sequential + + # Require alternatives section + require_alternatives: true + + # Require validation criteria + require_validation: true + +# Member configuration +members: + # Path to members.yml file (relative to .architecture) + definition_file: members.yml + + # Allow dynamic member creation + # When true, new members can be added during reviews if expertise is needed + allow_dynamic_creation: true + +# Templates configuration +templates: + # Directory containing templates (relative to .architecture) + directory: templates + + # Auto-populate template metadata + auto_populate_metadata: true + +# Output configuration +output: + # Verbosity level: minimal | normal | detailed + verbosity: normal + + # Include reasoning in outputs + include_reasoning: true + + # Format: markdown | json | yaml + format: markdown + +# Version tracking +version: + # Current framework version + framework_version: "1.2.0" + + # Project architecture version (update with each major architecture change) + architecture_version: "1.0.0" diff --git a/.architecture/templates/deferrals.md b/.architecture/templates/deferrals.md new file mode 100644 index 00000000..a67de6bb --- /dev/null +++ b/.architecture/templates/deferrals.md @@ -0,0 +1,338 @@ +# Deferred Architectural Decisions + +This document tracks architectural features, patterns, and complexity that have been consciously deferred for future implementation. Each deferral includes the rationale for waiting and clear trigger conditions for when it should be reconsidered. + +## Status Key + +- **Deferred**: Decision to defer is active, watching for trigger +- **Triggered**: Trigger condition met, needs implementation +- **Implemented**: Feature has been implemented (moved to ADR) +- **Cancelled**: No longer needed or relevant + +--- + +## Deferred Decisions + +### [Feature/Pattern Name] + +**Status**: Deferred +**Deferred Date**: YYYY-MM-DD +**Category**: [Architecture | Performance | Testing | Infrastructure | Security] +**Priority**: [Low | Medium | High] + +**What Was Deferred**: +[Brief description of the feature, pattern, or complexity that was deferred] + +**Original Proposal**: +[What was originally suggested - can quote from review or ADR] + +**Rationale for Deferring**: +- Current need score: [0-10] +- Complexity score: [0-10] +- Cost of waiting: [Low | Medium | High] +- Why deferring makes sense: [Explanation] + +**Simpler Current Approach**: +[What we're doing instead for now] + +**Trigger Conditions** (Implement when): +- [ ] [Specific condition 1 - make this measurable] +- [ ] [Specific condition 2] +- [ ] [Specific condition 3] + +**Implementation Notes**: +[Notes for when this is implemented - gotchas, considerations, references] + +**Related Documents**: +- [Link to ADR or review] +- [Link to discussion] + +**Last Reviewed**: YYYY-MM-DD + +--- + +## Example Entries + +### OAuth2 and SAML Authentication + +**Status**: Deferred +**Deferred Date**: 2024-01-15 +**Category**: Security +**Priority**: Medium + +**What Was Deferred**: +Support for OAuth2 and SAML authentication providers in addition to JWT + +**Original Proposal**: +Security Specialist recommended implementing comprehensive authentication middleware with support for OAuth2, SAML, and JWT with pluggable providers. + +**Rationale for Deferring**: +- Current need score: 3/10 (only JWT needed now) +- Complexity score: 8/10 (multiple abstractions, provider interfaces) +- Cost of waiting: Low +- All current requirements satisfied by JWT +- No confirmed need for other providers +- Can add later without major refactoring + +**Simpler Current Approach**: +JWT-only authentication with standard Express middleware. Clean interface makes adding other providers feasible later. + +**Trigger Conditions** (Implement when): +- [ ] Confirmed partner integration requiring OAuth2 +- [ ] Enterprise customer requiring SAML +- [ ] Three or more requests for alternative auth methods +- [ ] Strategic decision to support social login + +**Implementation Notes**: +When implementing: +- Extract current JWT logic into provider interface +- Ensure passport.js strategies are compatible +- Consider Auth0 or similar service instead of custom implementation +- Update user model to track auth provider + +**Related Documents**: +- ADR-005: Authentication System +- Review 1.0.0: Security Specialist section + +**Last Reviewed**: 2024-01-15 + +--- + +### Redis Caching Layer + +**Status**: Deferred +**Deferred Date**: 2024-01-15 +**Category**: Performance +**Priority**: Low + +**What Was Deferred**: +Redis-based caching layer with cache invalidation strategy + +**Original Proposal**: +Performance Specialist recommended implementing Redis caching for database queries, API responses, and session data with intelligent cache invalidation. + +**Rationale for Deferring**: +- Current need score: 2/10 (no performance problems) +- Complexity score: 9/10 (cache invalidation is hard) +- Cost of waiting: Zero until we have performance problems +- No evidence of slow operations +- Premature optimization +- Can optimize specific bottlenecks when they appear + +**Simpler Current Approach**: +No caching. Using database directly with optimized queries. Monitoring response times for degradation. + +**Trigger Conditions** (Implement when): +- [ ] Average response time > 500ms for key endpoints +- [ ] 95th percentile response time > 2 seconds +- [ ] Database CPU consistently > 70% +- [ ] Specific queries identified as bottlenecks via profiling +- [ ] Traffic exceeds 1000 req/sec + +**Implementation Notes**: +When implementing: +1. Profile first to identify actual bottlenecks +2. Start with simple in-memory cache for hot paths +3. Only add Redis if in-memory insufficient +4. Use cache-aside pattern +5. Consider read replicas before caching + +**Related Documents**: +- Review 1.0.0: Performance Specialist section +- Performance monitoring dashboard: [link] + +**Last Reviewed**: 2024-01-15 + +--- + +### Comprehensive Integration Test Suite + +**Status**: Deferred +**Deferred Date**: 2024-01-15 +**Category**: Testing +**Priority**: Medium + +**What Was Deferred**: +Full integration test suite with contract tests and property-based tests + +**Original Proposal**: +Maintainability Expert recommended comprehensive testing: unit tests, integration tests, contract tests, E2E tests, and property-based tests for all business logic. + +**Rationale for Deferring**: +- Current need score: 6/10 (tests needed but not all types) +- Complexity score: 8/10 (multiple frameworks, test data) +- Cost of waiting: Low (can add incrementally) +- Basic unit tests provide current value +- Can add test types as problems emerge +- Avoid testing implementation details that will change + +**Simpler Current Approach**: +Focused unit tests for business logic core and smoke tests for critical user flows. ~70% coverage on core domain logic. + +**Trigger Conditions** (Implement when): +- [ ] Integration failures between services (→ add integration tests) +- [ ] User-reported bugs in E2E flows (→ add E2E tests) +- [ ] Edge case bugs in business logic (→ add property tests) +- [ ] Contract breaking changes between teams (→ add contract tests) + +**Implementation Notes**: +When implementing: +- Add test types incrementally based on actual failures +- Integration tests: Start with most critical integration points +- E2E tests: Cover top 5 user journeys first +- Property tests: Focus on business logic with complex invariants +- Track which test types provide most value + +**Related Documents**: +- Review 1.0.0: Maintainability Expert section +- Testing strategy: [link] + +**Last Reviewed**: 2024-01-15 + +--- + +### Service Mesh (Istio/Linkerd) + +**Status**: Deferred +**Deferred Date**: 2024-01-15 +**Category**: Architecture +**Priority**: Low + +**What Was Deferred**: +Service mesh for microservices communication, observability, and traffic management + +**Original Proposal**: +Systems Architect recommended implementing service mesh (Istio or Linkerd) for service-to-service communication with circuit breakers, retries, and distributed tracing. + +**Rationale for Deferring**: +- Current need score: 3/10 (3 services, simple communication) +- Complexity score: 9/10 (major infrastructure, learning curve) +- Cost of waiting: Near zero (can add when needed) +- Current service count doesn't justify mesh complexity +- Simple HTTP with retries sufficient for now +- Major operational overhead for small benefit + +**Simpler Current Approach**: +Direct service-to-service HTTP calls with exponential backoff retry logic. Using standard HTTP health checks. + +**Trigger Conditions** (Implement when): +- [ ] Service count exceeds 10 +- [ ] Need circuit breakers for multiple services +- [ ] Need mTLS between services for security +- [ ] Need advanced traffic routing (canary, blue-green) +- [ ] Debugging service communication becomes difficult +- [ ] Need unified observability across services + +**Implementation Notes**: +When implementing: +- Evaluate Istio vs. Linkerd vs. newer alternatives +- Start with observability features first +- Phase in mTLS gradually +- Consider managed service mesh offerings +- Ensure team has time for learning curve +- Budget for operational overhead + +**Related Documents**: +- Review 2.0.0: Systems Architect section +- Current architecture diagram: [link] + +**Last Reviewed**: 2024-01-15 + +--- + +### Event Sourcing Architecture + +**Status**: Deferred +**Deferred Date**: 2024-01-15 +**Category**: Architecture +**Priority**: Low + +**What Was Deferred**: +Event sourcing pattern for capturing all changes as events + +**Original Proposal**: +Domain Expert recommended event sourcing to maintain complete audit trail and enable temporal queries. + +**Rationale for Deferring**: +- Current need score: 4/10 (audit trail needed, but basic version sufficient) +- Complexity score: 9/10 (major architectural change) +- Cost of waiting: Low (can add audit logging incrementally) +- Simple audit logging meets current compliance requirements +- Event sourcing is a one-way door decision +- Complexity of event sourcing not justified by current needs + +**Simpler Current Approach**: +Standard CRUD with audit logging table tracking changes to critical entities. Includes who, what, when for key operations. + +**Trigger Conditions** (Implement when): +- [ ] Need to reconstruct entity state at any point in time +- [ ] Current audit logging doesn't meet compliance requirements +- [ ] Need to replay events for new projections +- [ ] Temporal queries become core business requirement +- [ ] Need event-driven architecture for system integration + +**Implementation Notes**: +When implementing: +- Start with single bounded context, not whole system +- Choose event store carefully (EventStore, Kafka, custom) +- Plan migration strategy carefully +- Consider CQRS pattern alongside +- Ensure team understands event modeling +- This is a major architectural shift - not to be taken lightly + +**Related Documents**: +- Review 1.0.0: Domain Expert section +- Audit requirements: [link] +- Event sourcing investigation: [link] + +**Last Reviewed**: 2024-01-15 + +--- + +## Review Process + +This document should be reviewed: + +**Monthly**: Check for triggered conditions +- Review each deferred item +- Check if any trigger conditions have been met +- Update priority if context has changed + +**Quarterly**: Re-evaluate deferrals +- Are deferred items still relevant? +- Have requirements changed? +- Should priority be adjusted? +- Can any be cancelled? + +**During Architecture Reviews**: Reference deferrals +- Check if new features relate to deferrals +- Consider if triggered conditions met +- Avoid re-proposing already-deferred items +- Update relevant entries + +**When Triggers Met**: +1. Update status to "Triggered" +2. Create or update ADR for implementation +3. Plan implementation in next sprint/release +4. Reference this deferral in the ADR +5. After implementation, update status to "Implemented" + +## Metrics + +Track deferral outcomes to improve decision-making: + +| Metric | Value | Notes | +|--------|-------|-------| +| Total deferrals | [count] | All-time count | +| Active deferrals | [count] | Currently deferred | +| Triggered awaiting implementation | [count] | Need to address | +| Implemented | [count] | Were eventually needed | +| Cancelled | [count] | Were never needed | +| Average time before trigger | [days] | How long before we needed it | +| Hit rate (implemented/total) | [%] | How often deferred things are needed | + +**Target**: < 40% hit rate (most deferred things remain unneeded, validating deferral decisions) + +--- + +*This template should be copied to `.architecture/deferrals.md` in each project using pragmatic mode.* diff --git a/.architecture/templates/implementation_roadmap.md b/.architecture/templates/implementation_roadmap.md new file mode 100644 index 00000000..86d73e45 --- /dev/null +++ b/.architecture/templates/implementation_roadmap.md @@ -0,0 +1,125 @@ +# Implementation Roadmap for Version X.Y.Z + +## Overview + +This document outlines the implementation plan for architectural changes identified in the recalibration plan for version X.Y.Z. It breaks down high-level architectural changes into implementable tasks, assigns them to specific versions, and establishes acceptance criteria. + +## Target Versions + +This roadmap covers the following versions: +- **X.Y.Z**: [Brief description of focus] +- **X.Y.(Z+1)**: [Brief description of focus] +- **X.(Y+1).0**: [Brief description of focus] + +## Implementation Areas + +### [Area 1: e.g., Component Decomposition] + +**Overall Goal**: [Describe the high-level architectural goal for this area] + +#### Tasks for Version X.Y.Z + +| Task ID | Description | Dependencies | Complexity | Owner | Tests Required | +|---------|-------------|--------------|------------|-------|----------------| +| [A1.1] | [Detailed task description] | [Dependencies] | [Low/Medium/High] | [Owner] | [Test requirements] | +| [A1.2] | [...] | [...] | [...] | [...] | [...] | + +**Acceptance Criteria**: +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [...] + +#### Tasks for Version X.Y.(Z+1) + +| Task ID | Description | Dependencies | Complexity | Owner | Tests Required | +|---------|-------------|--------------|------------|-------|----------------| +| [A1.3] | [Detailed task description] | [Dependencies] | [Low/Medium/High] | [Owner] | [Test requirements] | +| [A1.4] | [...] | [...] | [...] | [...] | [...] | + +**Acceptance Criteria**: +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [...] + +### [Area 2: e.g., Security Enhancements] + +**Overall Goal**: [Describe the high-level architectural goal for this area] + +#### Tasks for Version X.Y.Z + +| Task ID | Description | Dependencies | Complexity | Owner | Tests Required | +|---------|-------------|--------------|------------|-------|----------------| +| [B1.1] | [Detailed task description] | [Dependencies] | [Low/Medium/High] | [Owner] | [Test requirements] | +| [B1.2] | [...] | [...] | [...] | [...] | [...] | + +**Acceptance Criteria**: +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [...] + +## Implementation Approach + +### Breaking vs. Non-Breaking Changes + +[Describe the approach to handling breaking changes, including deprecation policy, backward compatibility strategies, etc.] + +### Feature Flags + +[Document any feature flags that will be used to control the rollout of new architectural components] + +| Flag Name | Purpose | Default Value | Removal Version | +|-----------|---------|---------------|-----------------| +| [Flag name] | [Purpose] | [true/false] | [Version] | + +### Migration Support + +[Detail any migration utilities, scripts, or guidance that will be provided to help users adapt to architectural changes] + +## Testing Strategy + +### Component Tests + +[Describe the approach to testing individual architectural components] + +### Integration Tests + +[Describe the approach to testing integration between components] + +### Migration Tests + +[Describe the approach to testing migration paths from previous versions] + +## Documentation Plan + +| Document | Update Required | Responsible | Deadline | +|----------|-----------------|-------------|----------| +| [Document name] | [Description of update needed] | [Responsible person] | [Date] | + +## Risk Assessment + +| Risk | Impact | Likelihood | Mitigation Strategy | +|------|--------|------------|---------------------| +| [Risk description] | [High/Medium/Low] | [High/Medium/Low] | [Mitigation approach] | + +## Timeline + +| Milestone | Target Date | Dependencies | Owner | +|-----------|-------------|--------------|-------| +| [Milestone description] | [Date] | [Dependencies] | [Owner] | + +## Progress Tracking + +Progress on this implementation roadmap will be tracked in: +- [Link to tracking tool/document] +- [Link to relevant GitHub projects/issues] + +## Appendices + +### A. Architecture Diagrams + +[Include or link to relevant architecture diagrams] + +### B. Relevant ADRs + +- [ADR-XXX: Title](link-to-adr) +- [ADR-YYY: Title](link-to-adr) \ No newline at end of file diff --git a/.architecture/templates/progress_tracking.md b/.architecture/templates/progress_tracking.md new file mode 100644 index 00000000..4071c213 --- /dev/null +++ b/.architecture/templates/progress_tracking.md @@ -0,0 +1,89 @@ +# Architectural Changes Progress Tracking + +## Overview + +This document tracks the implementation progress of architectural changes identified in the recalibration plan for version X.Y.Z. It is updated regularly to reflect current status and any adjustments to the implementation approach. + +**Last Updated**: [DATE] + +## Executive Summary + +| Category | Total Items | Completed | In Progress | Not Started | Deferred | +|----------|-------------|-----------|-------------|-------------|----------| +| Architectural Changes | [Number] | [Number] | [Number] | [Number] | [Number] | +| Implementation Improvements | [Number] | [Number] | [Number] | [Number] | [Number] | +| Documentation Enhancements | [Number] | [Number] | [Number] | [Number] | [Number] | +| Process Adjustments | [Number] | [Number] | [Number] | [Number] | [Number] | +| **TOTAL** | [Number] | [Number] | [Number] | [Number] | [Number] | + +**Completion Percentage**: [X%] + +## Detailed Status + +### Architectural Changes + +| ID | Recommendation | Priority | Status | Target Version | Actual Version | Notes | +|----|---------------|----------|--------|----------------|----------------|-------| +| A1 | [Brief description] | [Priority] | [Not Started/In Progress/Completed/Deferred] | [Version] | [Actual version or N/A] | [Current status notes] | + +### Implementation Improvements + +| ID | Recommendation | Priority | Status | Target Version | Actual Version | Notes | +|----|---------------|----------|--------|----------------|----------------|-------| +| I1 | [Brief description] | [Priority] | [Not Started/In Progress/Completed/Deferred] | [Version] | [Actual version or N/A] | [Current status notes] | + +### Documentation Enhancements + +| ID | Recommendation | Priority | Status | Target Version | Actual Version | Notes | +|----|---------------|----------|--------|----------------|----------------|-------| +| D1 | [Brief description] | [Priority] | [Not Started/In Progress/Completed/Deferred] | [Version] | [Actual version or N/A] | [Current status notes] | + +### Process Adjustments + +| ID | Recommendation | Priority | Status | Target Version | Actual Version | Notes | +|----|---------------|----------|--------|----------------|----------------|-------| +| P1 | [Brief description] | [Priority] | [Not Started/In Progress/Completed/Deferred] | [Version] | [Actual version or N/A] | [Current status notes] | + +## Implementation Adjustments + +This section documents any adjustments made to the implementation approach since the original recalibration plan. + +| ID | Original Approach | Adjusted Approach | Rationale | Impact | +|----|-------------------|-------------------|-----------|--------| +| [ID] | [Description] | [Description] | [Reason for change] | [Impact of change] | + +## Milestone Progress + +| Milestone | Target Date | Status | Actual/Projected Completion | Notes | +|-----------|-------------|--------|---------------------------|-------| +| [Milestone] | [Date] | [Not Started/In Progress/Completed/Delayed] | [Date] | [Notes] | + +## Blocked Items + +| ID | Blocker Description | Impact | Owner | Resolution Plan | Projected Resolution Date | +|----|---------------------|--------|-------|-----------------|---------------------------| +| [ID] | [Description] | [Impact] | [Owner] | [Plan] | [Date] | + +## Recently Completed Items + +| ID | Description | Completion Date | Implemented In | Implementation Notes | +|----|-------------|-----------------|----------------|----------------------| +| [ID] | [Description] | [Date] | [Version] | [Notes] | + +## Next Check-in + +The next progress check-in meeting is scheduled for [DATE]. + +## Appendices + +### A. Test Coverage Report + +[Summary of test coverage for implemented architectural changes] + +### B. Documentation Status + +[Summary of documentation updates status] + +### C. Quality Metrics + +[Key quality metrics for implemented architectural changes] \ No newline at end of file diff --git a/.architecture/templates/recalibration_plan.md b/.architecture/templates/recalibration_plan.md new file mode 100644 index 00000000..25ef7af2 --- /dev/null +++ b/.architecture/templates/recalibration_plan.md @@ -0,0 +1,70 @@ +# Architectural Recalibration Plan: Version X.Y.Z + +## Overview + +This document outlines the action plan derived from the architectural review of version X.Y.Z. It categorizes and prioritizes recommendations to guide implementation across upcoming releases. + +## Review Summary + +- Review Date: [DATE] +- Review Document: [LINK TO REVIEW] +- Participants: [LIST OF PARTICIPANTS] + +## Action Items + +### Architectural Changes + +| ID | Recommendation | Priority | Owner | Target Version | Dependencies | Notes | +|----|---------------|----------|-------|----------------|--------------|-------| +| A1 | [Brief description] | [Critical/High/Medium/Low] | [Owner] | [Version] | [Dependencies] | [Additional context] | + +### Implementation Improvements + +| ID | Recommendation | Priority | Owner | Target Version | Dependencies | Notes | +|----|---------------|----------|-------|----------------|--------------|-------| +| I1 | [Brief description] | [Critical/High/Medium/Low] | [Owner] | [Version] | [Dependencies] | [Additional context] | + +### Documentation Enhancements + +| ID | Recommendation | Priority | Owner | Target Version | Dependencies | Notes | +|----|---------------|----------|-------|----------------|--------------|-------| +| D1 | [Brief description] | [Critical/High/Medium/Low] | [Owner] | [Version] | [Dependencies] | [Additional context] | + +### Process Adjustments + +| ID | Recommendation | Priority | Owner | Target Version | Dependencies | Notes | +|----|---------------|----------|-------|----------------|--------------|-------| +| P1 | [Brief description] | [Critical/High/Medium/Low] | [Owner] | [Version] | [Dependencies] | [Additional context] | + +## Technical Debt Items + +Items identified in the review that won't be addressed immediately but should be tracked: + +| ID | Description | Impact | Potential Resolution Timeframe | +|----|-------------|--------|--------------------------------| +| TD1 | [Description] | [Low/Medium/High] | [Timeframe] | + +## Decision Records + +List of Architectural Decision Records (ADRs) that need to be created or updated based on the review: + +| ADR ID | Title | Status | Owner | Target Completion | +|--------|-------|--------|-------|-------------------| +| ADR-XXX | [Title] | [Draft/Proposed/Accepted] | [Owner] | [Date] | + +## Timeline + +Overview of the recalibration implementation timeline: + +- Analysis & Prioritization: [Start Date] - [End Date] +- Architectural Plan Update: [Start Date] - [End Date] +- Documentation Refresh: [Start Date] - [End Date] +- Implementation Roadmapping: [Start Date] - [End Date] + +## Next Steps + +Immediate next actions to be taken: + +1. [Action 1] +2. [Action 2] +3. [Action 3] \ No newline at end of file diff --git a/.architecture/templates/review-template.md b/.architecture/templates/review-template.md new file mode 100644 index 00000000..a9a64344 --- /dev/null +++ b/.architecture/templates/review-template.md @@ -0,0 +1,374 @@ +# Architectural Review: Version X.Y.Z + +## Overview + +This document contains the comprehensive architectural review for version X.Y.Z, conducted from [START DATE] to [END DATE]. The review evaluates the current architecture against project goals, industry best practices, and future requirements. + +## Review Details + +- **Version Reviewed**: X.Y.Z +- **Review Period**: [START DATE] - [END DATE] +- **Review Team**: [NAMES OR ROLES] +- **Review Methodology**: [DESCRIPTION OF REVIEW PROCESS] + +## Senior Thinking Checklist + +> **Note**: This checklist frames the architectural review by externalizing the "silent questions" that senior engineers ask before diving into technical details. Each reviewer should consider these questions from their specialist perspective. + +Before diving into detailed strengths and weaknesses, reviewers should address these framing questions: + +### Change Characterization +- **Nature of changes in this version**: Are we introducing new ideas, or expressing existing ones in new places? +- **Architectural depth**: Do changes belong at the surface (API, UX) or deep in the system? +- **Conceptual alignment**: How well do changes align with existing architectural patterns and domain model? + +### Spread Analysis +- **Pattern propagation**: If the patterns/approaches in this version spread across the codebase, is that desirable or a liability? +- **Consistency impact**: Do these changes establish patterns we want repeated or should be contained? +- **Technical debt trajectory**: Are we creating consistency or divergence? + +### Blast Radius Assessment +- **Failure scope**: What's the blast radius if these changes are wrong and need to be undone? +- **Component coupling**: Which components/services/modules are directly affected? +- **Team impact**: Which teams need to coordinate if changes fail or evolve? +- **User impact**: What's the user-facing risk if this goes wrong? + +### Reversibility Analysis +- **Rollback feasibility**: How easily can these changes be reversed if needed? +- **Migration complexity**: What migration paths exist for reverting or evolving? +- **Option preservation**: Do these changes keep future options open or close doors? +- **Commitment level**: Are we making reversible decisions or permanent commitments? + +### Timing & Sequencing +- **System readiness**: Is the system technically ready for these changes? + - Do we have adequate observability? + - Are dependencies stable? + - Is infrastructure sufficient? +- **Team readiness**: Is the team prepared for these changes? + - Do they understand the patterns being introduced? + - Do they have necessary skills? + - Is documentation adequate? +- **Sequencing concerns**: Are there prerequisites that should have come first? +- **Coordination needs**: What cross-team coordination is required? + +### Social Cost Evaluation +- **Complexity burden**: Will these changes confuse more people than they help? +- **Learning curve**: What's the onboarding impact for new team members? +- **Cognitive load**: How much mental overhead do these patterns add? +- **Documentation needs**: What explanation is required for maintainability? + +### Confidence Assessment +- **Model correctness**: Are we confident in the architectural model, not just the implementation? +- **Test coverage**: Could tests pass while the model is still wrong? +- **Assumption validation**: What assumptions underpin these changes? How validated are they? +- **Edge cases**: What edge cases might not be captured by current testing? + +### Overall Architectural Direction +- **Alignment**: Do these changes move us toward or away from target architecture? +- **Evolution path**: Do they represent progress on our architectural evolution journey? +- **Technical debt**: Do they add to, reduce, or shift technical debt? + +--- + +*Each specialist reviewer should consider these questions through their domain lens before providing detailed technical analysis.* + +## Executive Summary + +[A concise summary of the review's key findings, major recommendations, and overall architectural health assessment. This should highlight critical areas that need attention and strengths to preserve.] + +**Overall Architecture Health**: [Excellent/Good/Fair/Concerning/Critical] + +**Key Strengths**: +- [Strength 1] +- [Strength 2] +- [Strength 3] + +**Critical Concerns**: +- [Concern 1] +- [Concern 2] +- [Concern 3] + +## Individual Perspectives + +### Systems Architect Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Domain Expert Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Security Specialist Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Maintainability Expert Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Performance Specialist Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Implementation Strategist Review + +**Reviewer**: [NAME] + +**Blast Radius Analysis**: +- [Assessment of change impact scope] +- [Components/teams/users affected] +- [Risk level and mitigation strategies] + +**Reversibility Assessment**: +- [How easily changes can be rolled back] +- [Migration paths for evolution or reversal] +- [Options preserved vs. doors closed] + +**Timing & Readiness**: +- **System Readiness**: [Technical readiness assessment] +- **Team Readiness**: [Skill and understanding assessment] +- **Sequencing Concerns**: [Prerequisites or coordination needs] + +**Social Cost**: +- [Learning curve and cognitive load impact] +- [Clarity vs. confusion trade-offs] +- [Documentation and training needs] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### AI Engineer Review + +**Reviewer**: [NAME] + +**Strengths**: +- [Strength 1] +- [Strength 2] +- [...] + +**Weaknesses**: +- [Weakness 1] +- [Weakness 2] +- [...] + +**Recommendations**: +- [Recommendation 1] +- [Recommendation 2] +- [...] + +### Pragmatic Enforcer Review + +**Reviewer**: [NAME] +**Mode**: [Strict | Balanced | Lenient] + +**Note**: *This section only appears when pragmatic_mode is enabled in `.architecture/config.yml`* + +**Overall Simplicity Assessment**: +[High-level assessment of whether the architecture and recommendations maintain appropriate simplicity or show signs of over-engineering] + +**Strengths**: +- [Areas where simplicity is maintained well] +- [Good examples of appropriate complexity] +- [Well-justified abstractions] + +**Concerns**: +- [Areas of potentially unnecessary complexity] +- [Abstractions that may be premature] +- [Features that might be YAGNI violations] + +**Challenges to Recommendations**: + +#### Challenge to [Architect Role] + +**Original Recommendation**: "[Quote the recommendation being challenged]" + +**Necessity Assessment**: [Score 0-10] +- **Current need**: [Why this is/isn't needed right now] +- **Future need**: [Likelihood and timeframe for actual need] +- **Cost of waiting**: [What happens if we defer this] + +**Complexity Assessment**: [Score 0-10] +- **Added complexity**: [What complexity this introduces] +- **Maintenance burden**: [Ongoing cost to maintain] +- **Learning curve**: [Impact on team/new developers] + +**Simpler Alternative**: +[Concrete proposal for a simpler approach that meets current actual requirements] + +**Recommendation**: [✅ Implement now | ⚠️ Implement simplified version | ⏸️ Defer until needed | ❌ Skip entirely] + +**Justification**: +[Clear reasoning for the recommendation, balancing current needs vs future flexibility] + +--- + +*[Repeat the Challenge section for each significant recommendation from other architects]* + +**Summary Recommendations**: +1. [Key recommendation 1 with action: implement/simplify/defer/skip] +2. [Key recommendation 2 with action] +3. [...] + +**Deferred Decisions** (tracked in `.architecture/deferrals.md`): +- [Feature/pattern being deferred] → Trigger: [condition] +- [Feature/pattern being deferred] → Trigger: [condition] + +## Collaborative Analysis + +This section reflects the consensus reached after cross-functional discussion of individual findings. + +### Consolidated Strengths + +1. [Strength 1] - [Brief explanation] +2. [Strength 2] - [Brief explanation] +3. [...] + +### Consolidated Weaknesses + +1. [Weakness 1] - [Brief explanation] +2. [Weakness 2] - [Brief explanation] +3. [...] + +### Prioritized Improvements + +**Critical (Address in next release)**: +1. [Improvement 1] +2. [Improvement 2] + +**High (Address within next 2 releases)**: +1. [Improvement 3] +2. [Improvement 4] + +**Medium (Address within next 3-4 releases)**: +1. [Improvement 5] +2. [Improvement 6] + +**Low (Address as resources permit)**: +1. [Improvement 7] +2. [Improvement 8] + +## Technical Debt Assessment + +| Area | Current Debt Level | Trend | Impact | Notes | +|------|-------------------|-------|--------|-------| +| [Area 1] | [High/Medium/Low] | [Increasing/Stable/Decreasing] | [High/Medium/Low] | [Notes] | +| [Area 2] | [...] | [...] | [...] | [...] | + +## Architectural Evolution + +### Current Architecture vs. Target Architecture + +[Describe the gap between the current architecture and where it needs to be in the future. Include diagrams if helpful.] + +### Migration Path + +[Outline a high-level path for evolving from the current architecture to the target state, considering backward compatibility, phasing, and risk mitigation.] + +## Conclusion + +[Summarize the review's key points and provide a clear path forward. Emphasize both immediate actions and longer-term directions.] + +## Appendices + +### A. Review Methodology + +[Detailed description of the review process, including any tools, frameworks, or metrics used.] + +### B. Architecture Diagrams + +[Current architecture diagrams and any proposed future state diagrams.] + +### C. Metrics Analysis + +[Quantitative analysis of architectural metrics, if available.] + +### D. Referenced Documents + +[List of documents referenced during the review.] \ No newline at end of file diff --git a/.architecture/templates/version_comparison.md b/.architecture/templates/version_comparison.md new file mode 100644 index 00000000..772867ca --- /dev/null +++ b/.architecture/templates/version_comparison.md @@ -0,0 +1,156 @@ +# Version Comparison: [Old Version] to [New Version] + +## Overview + +This document provides a comprehensive comparison between version [Old Version] and [New Version], focusing on architectural changes, their impact, and migration considerations. + +## Architectural Changes + +### Major Changes + +| ID | Description | ADR Reference | Impact Level | +|----|-------------|--------------|--------------| +| M1 | [Brief description of major change] | [ADR-XXX](link-to-adr) | [High/Medium/Low] | +| M2 | [...] | [...] | [...] | + +### Minor Changes + +| ID | Description | ADR Reference | Impact Level | +|----|-------------|--------------|--------------| +| m1 | [Brief description of minor change] | [ADR-XXX](link-to-adr) or N/A | [High/Medium/Low] | +| m2 | [...] | [...] | [...] | + +### Deprecated Components + +| Component | Replacement | Deprecation Notice Added | Planned Removal Version | +|-----------|-------------|--------------------------|-------------------------| +| [Component name] | [Replacement component] | [Yes/No] | [Version] | + +## Before/After Analysis + +### [Component/Area 1] + +**Before:** +``` +[Diagram or code snippet showing the component before changes] +``` + +**After:** +``` +[Diagram or code snippet showing the component after changes] +``` + +**Key Differences:** +* [Difference 1] +* [Difference 2] +* [...] + +### [Component/Area 2] + +**Before:** +``` +[Diagram or code snippet showing the component before changes] +``` + +**After:** +``` +[Diagram or code snippet showing the component after changes] +``` + +**Key Differences:** +* [Difference 1] +* [Difference 2] +* [...] + +## Impact Analysis + +### Developer Experience + +| Aspect | Change | Impact | +|--------|--------|--------| +| API Usability | [Improved/Degraded/Unchanged] | [Description of impact] | +| Documentation | [Improved/Degraded/Unchanged] | [Description of impact] | +| Development Workflow | [Improved/Degraded/Unchanged] | [Description of impact] | +| Testing | [Improved/Degraded/Unchanged] | [Description of impact] | + +### Performance Characteristics + +| Metric | Before | After | Change (%) | Notes | +|--------|--------|-------|-----------|-------| +| [Metric 1] | [Value] | [Value] | [%] | [Notes] | +| [Metric 2] | [...] | [...] | [...] | [...] | + +### Security Posture + +| Security Aspect | Change | Impact | +|-----------------|--------|--------| +| Authentication | [Improved/Degraded/Unchanged] | [Description of impact] | +| Authorization | [Improved/Degraded/Unchanged] | [Description of impact] | +| Data Protection | [Improved/Degraded/Unchanged] | [Description of impact] | +| Attack Surface | [Reduced/Increased/Unchanged] | [Description of impact] | + +### Maintainability Metrics + +| Metric | Before | After | Change | Notes | +|--------|--------|-------|--------|-------| +| Code Complexity | [Value] | [Value] | [Change] | [Notes] | +| Test Coverage | [Value] | [Value] | [Change] | [Notes] | +| Coupling | [Value] | [Value] | [Change] | [Notes] | +| Cohesion | [Value] | [Value] | [Change] | [Notes] | + +### Observability Capabilities + +| Capability | Before | After | Change | Notes | +|------------|--------|-------|--------|-------| +| Logging | [Description] | [Description] | [Improved/Degraded/Unchanged] | [Notes] | +| Metrics | [Description] | [Description] | [Improved/Degraded/Unchanged] | [Notes] | +| Tracing | [Description] | [Description] | [Improved/Degraded/Unchanged] | [Notes] | +| Alerting | [Description] | [Description] | [Improved/Degraded/Unchanged] | [Notes] | + +## Implementation Deviations + +This section documents where the implemented changes differed from the original recommendations in the architectural review and recalibration plan. + +| ID | Original Recommendation | Actual Implementation | Rationale | +|----|-------------------------|----------------------|-----------| +| [ID] | [Description] | [Description] | [Explanation] | + +## Migration Guide + +### Breaking Changes + +| Change | Migration Path | Complexity | Tools Available | +|--------|---------------|------------|-----------------| +| [Change description] | [How to migrate] | [High/Medium/Low] | [Tools or scripts] | + +### Update Steps + +1. [Step 1] +2. [Step 2] +3. [...] + +### Code Examples + +**Before:** +``` +[Code example before migration] +``` + +**After:** +``` +[Code example after migration] +``` + +## Appendices + +### A. Performance Test Results + +[Detailed performance test results] + +### B. Security Assessment + +[Security assessment details] + +### C. User Feedback + +[Summary of any user feedback on the changes] \ No newline at end of file diff --git a/.gemini/settings.json b/.gemini/settings.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/.gemini/settings.json @@ -0,0 +1 @@ +{} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index eb8c1865..1fdea4aa 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -12,6 +12,13 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 + - name: Verify Namespace Packages + run: | + if [ -f "core/opl/__init__.py" ] || [ -f "extras/opl/__init__.py" ]; then + echo "Error: __init__.py found in namespace package root (core/opl or extras/opl)." + echo "This breaks PEP 420 implicit namespace packaging across split directories." + exit 1 + fi - name: Super-Linter uses: github/super-linter@v4 env: diff --git a/DELME/aaa.json b/DELME/aaa.json new file mode 100644 index 00000000..2e29c882 --- /dev/null +++ b/DELME/aaa.json @@ -0,0 +1,12 @@ +{ + "a": 42, + "date": "2024-06-09T15:53:25+00:00", + "group": "Core Platforms", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-5096-duration.log", + "product": "Red Hat Satellite", + "release": "stream", + "result": "FAIL", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-5096-duration", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch" +} \ No newline at end of file diff --git a/DELME/aaa.py b/DELME/aaa.py new file mode 100755 index 00000000..e724f458 --- /dev/null +++ b/DELME/aaa.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + + +import csv +import unittest.mock + +HISTORY_NORMAL = """metric1,metric2 +5,1000 +6,995 +4,1000 +5,1005 +""" + + +# https://stackoverflow.com/questions/26783678/python-mock-builtin-open-in-a-class-using-two-different-files +def get_mock_open(files: dict[str, str]): + def my_mock_open(filename, *args, **kwargs): + if filename in files: + i = unittest.mock.mock_open(read_data=files[filename]).return_value + i.name = filename + return i + raise FileNotFoundError(f"(mock) Unable to open {filename}") + + return unittest.mock.MagicMock(side_effect=my_mock_open) + + +files = { + "/tmp/history.csv": HISTORY_NORMAL, +} + +#for i in [1]: +with unittest.mock.patch("builtins.open", get_mock_open(files)) as m: + with open("/tmp/history.csv", "r", newline="") as fd: + reader = csv.DictReader(fd) + for row in reader: + print(f"Processing {row}") diff --git a/DELME/aaa/01-manifest-excercise-ManifestDelete.log.rdd.json b/DELME/aaa/01-manifest-excercise-ManifestDelete.log.rdd.json new file mode 100644 index 00000000..84138f63 --- /dev/null +++ b/DELME/aaa/01-manifest-excercise-ManifestDelete.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:42:44+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/01-manifest-excercise-ManifestDelete.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "01-manifest-excercise-ManifestDelete", + "result": "FAIL" +} diff --git a/DELME/aaa/01-manifest-excercise-ManifestRefresh.log.rdd.json b/DELME/aaa/01-manifest-excercise-ManifestRefresh.log.rdd.json new file mode 100644 index 00000000..246fde8c --- /dev/null +++ b/DELME/aaa/01-manifest-excercise-ManifestRefresh.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:42:38+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/01-manifest-excercise-ManifestRefresh.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "01-manifest-excercise-ManifestRefresh", + "result": "FAIL" +} diff --git a/DELME/aaa/01-manifest-excercise-ManifestUpload.log.rdd.json b/DELME/aaa/01-manifest-excercise-ManifestUpload.log.rdd.json new file mode 100644 index 00000000..67ec3c53 --- /dev/null +++ b/DELME/aaa/01-manifest-excercise-ManifestUpload.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:42:29+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/01-manifest-excercise-ManifestUpload.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "01-manifest-excercise-ManifestUpload", + "result": "FAIL" +} diff --git a/DELME/aaa/09-lce-create-Pre.log.rdd.json b/DELME/aaa/09-lce-create-Pre.log.rdd.json new file mode 100644 index 00000000..251e10df --- /dev/null +++ b/DELME/aaa/09-lce-create-Pre.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:48:21+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/09-lce-create-Pre.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "09-lce-create-Pre", + "result": "FAIL" +} diff --git a/DELME/aaa/09-lce-create-Prod.log.rdd.json b/DELME/aaa/09-lce-create-Prod.log.rdd.json new file mode 100644 index 00000000..38bdb1fb --- /dev/null +++ b/DELME/aaa/09-lce-create-Prod.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:48:28+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/09-lce-create-Prod.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "09-lce-create-Prod", + "result": "PASS" +} diff --git a/DELME/aaa/09-lce-create-QA.log.rdd.json b/DELME/aaa/09-lce-create-QA.log.rdd.json new file mode 100644 index 00000000..4f636534 --- /dev/null +++ b/DELME/aaa/09-lce-create-QA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:48:14+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/09-lce-create-QA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "09-lce-create-QA", + "result": "PASS" +} diff --git a/DELME/aaa/09-lce-create-Test.log.rdd.json b/DELME/aaa/09-lce-create-Test.log.rdd.json new file mode 100644 index 00000000..65742f35 --- /dev/null +++ b/DELME/aaa/09-lce-create-Test.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:48:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/09-lce-create-Test.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "09-lce-create-Test", + "result": "PASS" +} diff --git a/DELME/aaa/12-repo-sync-rhel6.log.rdd.json b/DELME/aaa/12-repo-sync-rhel6.log.rdd.json new file mode 100644 index 00000000..961916f0 --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel6.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:48:56+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel6.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel6", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel7.log.rdd.json b/DELME/aaa/12-repo-sync-rhel7.log.rdd.json new file mode 100644 index 00000000..56766b5f --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel7.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T07:53:58+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel7.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel7", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel7extras.log.rdd.json b/DELME/aaa/12-repo-sync-rhel7extras.log.rdd.json new file mode 100644 index 00000000..044f50c4 --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel7extras.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:01:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel7extras.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel7extras", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel8appstream.log.rdd.json b/DELME/aaa/12-repo-sync-rhel8appstream.log.rdd.json new file mode 100644 index 00000000..bae7a498 --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel8appstream.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:05:35+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel8appstream.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel8appstream", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel8baseos.log.rdd.json b/DELME/aaa/12-repo-sync-rhel8baseos.log.rdd.json new file mode 100644 index 00000000..804a9952 --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel8baseos.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:01:45+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel8baseos.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel8baseos", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel9appstream.log.rdd.json b/DELME/aaa/12-repo-sync-rhel9appstream.log.rdd.json new file mode 100644 index 00000000..c8d762ed --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel9appstream.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:15:26+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel9appstream.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel9appstream", + "result": "FAIL" +} diff --git a/DELME/aaa/12-repo-sync-rhel9baseos.log.rdd.json b/DELME/aaa/12-repo-sync-rhel9baseos.log.rdd.json new file mode 100644 index 00000000..ac3f3719 --- /dev/null +++ b/DELME/aaa/12-repo-sync-rhel9baseos.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:13:40+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/12-repo-sync-rhel9baseos.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "12-repo-sync-rhel9baseos", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-create-rhel6-os.log.rdd.json b/DELME/aaa/13b-cv-create-rhel6-os.log.rdd.json new file mode 100644 index 00000000..40366e5c --- /dev/null +++ b/DELME/aaa/13b-cv-create-rhel6-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:19:25+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-create-rhel6-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-create-rhel6-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-create-rhel7-os.log.rdd.json b/DELME/aaa/13b-cv-create-rhel7-os.log.rdd.json new file mode 100644 index 00000000..da63362d --- /dev/null +++ b/DELME/aaa/13b-cv-create-rhel7-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:21:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-create-rhel7-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-create-rhel7-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-create-rhel8-os.log.rdd.json b/DELME/aaa/13b-cv-create-rhel8-os.log.rdd.json new file mode 100644 index 00000000..b9484293 --- /dev/null +++ b/DELME/aaa/13b-cv-create-rhel8-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:23:49+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-create-rhel8-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-create-rhel8-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-create-rhel9-os.log.rdd.json b/DELME/aaa/13b-cv-create-rhel9-os.log.rdd.json new file mode 100644 index 00000000..9985f1f9 --- /dev/null +++ b/DELME/aaa/13b-cv-create-rhel9-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:26:13+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-create-rhel9-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-create-rhel9-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-publish-rhel6-os.log.rdd.json b/DELME/aaa/13b-cv-publish-rhel6-os.log.rdd.json new file mode 100644 index 00000000..c40664d5 --- /dev/null +++ b/DELME/aaa/13b-cv-publish-rhel6-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:19:32+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-publish-rhel6-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-publish-rhel6-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-publish-rhel7-os.log.rdd.json b/DELME/aaa/13b-cv-publish-rhel7-os.log.rdd.json new file mode 100644 index 00000000..2cbc63c0 --- /dev/null +++ b/DELME/aaa/13b-cv-publish-rhel7-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:21:34+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-publish-rhel7-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-publish-rhel7-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-publish-rhel8-os.log.rdd.json b/DELME/aaa/13b-cv-publish-rhel8-os.log.rdd.json new file mode 100644 index 00000000..08bb28dc --- /dev/null +++ b/DELME/aaa/13b-cv-publish-rhel8-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:23:56+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-publish-rhel8-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-publish-rhel8-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13b-cv-publish-rhel9-os.log.rdd.json b/DELME/aaa/13b-cv-publish-rhel9-os.log.rdd.json new file mode 100644 index 00000000..9dba32c4 --- /dev/null +++ b/DELME/aaa/13b-cv-publish-rhel9-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:26:20+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13b-cv-publish-rhel9-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13b-cv-publish-rhel9-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-component-add-rhel6-os.log.rdd.json b/DELME/aaa/13c-ccv-component-add-rhel6-os.log.rdd.json new file mode 100644 index 00000000..2840a3e0 --- /dev/null +++ b/DELME/aaa/13c-ccv-component-add-rhel6-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:19:59+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-component-add-rhel6-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-component-add-rhel6-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-component-add-rhel7-os.log.rdd.json b/DELME/aaa/13c-ccv-component-add-rhel7-os.log.rdd.json new file mode 100644 index 00000000..1988dd2b --- /dev/null +++ b/DELME/aaa/13c-ccv-component-add-rhel7-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:22:03+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-component-add-rhel7-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-component-add-rhel7-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-component-add-rhel8-os.log.rdd.json b/DELME/aaa/13c-ccv-component-add-rhel8-os.log.rdd.json new file mode 100644 index 00000000..85a72650 --- /dev/null +++ b/DELME/aaa/13c-ccv-component-add-rhel8-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:24:26+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-component-add-rhel8-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-component-add-rhel8-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-component-add-rhel9-os.log.rdd.json b/DELME/aaa/13c-ccv-component-add-rhel9-os.log.rdd.json new file mode 100644 index 00000000..6f7734cf --- /dev/null +++ b/DELME/aaa/13c-ccv-component-add-rhel9-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:26:46+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-component-add-rhel9-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-component-add-rhel9-os", + "result": "PASS" +} diff --git a/DELME/aaa/13c-ccv-create-rhel6.log.rdd.json b/DELME/aaa/13c-ccv-create-rhel6.log.rdd.json new file mode 100644 index 00000000..a9a0d019 --- /dev/null +++ b/DELME/aaa/13c-ccv-create-rhel6.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:19:52+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-create-rhel6.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-create-rhel6", + "result": "PASS" +} diff --git a/DELME/aaa/13c-ccv-create-rhel7.log.rdd.json b/DELME/aaa/13c-ccv-create-rhel7.log.rdd.json new file mode 100644 index 00000000..143741b4 --- /dev/null +++ b/DELME/aaa/13c-ccv-create-rhel7.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:21:56+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-create-rhel7.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-create-rhel7", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-create-rhel8.log.rdd.json b/DELME/aaa/13c-ccv-create-rhel8.log.rdd.json new file mode 100644 index 00000000..b08d3399 --- /dev/null +++ b/DELME/aaa/13c-ccv-create-rhel8.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:24:19+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-create-rhel8.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-create-rhel8", + "result": "PASS" +} diff --git a/DELME/aaa/13c-ccv-create-rhel9.log.rdd.json b/DELME/aaa/13c-ccv-create-rhel9.log.rdd.json new file mode 100644 index 00000000..c99cd146 --- /dev/null +++ b/DELME/aaa/13c-ccv-create-rhel9.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:26:39+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-create-rhel9.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-create-rhel9", + "result": "PASS" +} diff --git a/DELME/aaa/13c-ccv-publish-rhel6-os.log.rdd.json b/DELME/aaa/13c-ccv-publish-rhel6-os.log.rdd.json new file mode 100644 index 00000000..99c0e498 --- /dev/null +++ b/DELME/aaa/13c-ccv-publish-rhel6-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:20:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-publish-rhel6-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-publish-rhel6-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-publish-rhel7-os.log.rdd.json b/DELME/aaa/13c-ccv-publish-rhel7-os.log.rdd.json new file mode 100644 index 00000000..2da37815 --- /dev/null +++ b/DELME/aaa/13c-ccv-publish-rhel7-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:22:11+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-publish-rhel7-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-publish-rhel7-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-publish-rhel8-os.log.rdd.json b/DELME/aaa/13c-ccv-publish-rhel8-os.log.rdd.json new file mode 100644 index 00000000..4c319735 --- /dev/null +++ b/DELME/aaa/13c-ccv-publish-rhel8-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:24:34+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-publish-rhel8-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-publish-rhel8-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13c-ccv-publish-rhel9-os.log.rdd.json b/DELME/aaa/13c-ccv-publish-rhel9-os.log.rdd.json new file mode 100644 index 00000000..15a4c3b0 --- /dev/null +++ b/DELME/aaa/13c-ccv-publish-rhel9-os.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:26:54+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13c-ccv-publish-rhel9-os.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13c-ccv-publish-rhel9-os", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel6-Pre.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel6-Pre.log.rdd.json new file mode 100644 index 00000000..cbd2e9b0 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel6-Pre.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:20:55+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel6-Pre.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel6-Pre", + "result": "PASS" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel6-Prod.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel6-Prod.log.rdd.json new file mode 100644 index 00000000..46ef8f14 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel6-Prod.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:21:08+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel6-Prod.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel6-Prod", + "result": "ERROR" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel6-QA.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel6-QA.log.rdd.json new file mode 100644 index 00000000..27325fdd --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel6-QA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:20:42+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel6-QA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel6-QA", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel6-Test.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel6-Test.log.rdd.json new file mode 100644 index 00000000..00630321 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel6-Test.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:20:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel6-Test.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel6-Test", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel7-Pre.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel7-Pre.log.rdd.json new file mode 100644 index 00000000..24a731f9 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel7-Pre.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:23:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel7-Pre.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel7-Pre", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel7-Prod.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel7-Prod.log.rdd.json new file mode 100644 index 00000000..0ebad8e4 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel7-Prod.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:23:25+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel7-Prod.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel7-Prod", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel7-QA.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel7-QA.log.rdd.json new file mode 100644 index 00000000..94e9600b --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel7-QA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:22:52+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel7-QA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel7-QA", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel7-Test.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel7-Test.log.rdd.json new file mode 100644 index 00000000..5a234915 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel7-Test.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:22:36+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel7-Test.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel7-Test", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel8-Pre.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel8-Pre.log.rdd.json new file mode 100644 index 00000000..96199cc0 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel8-Pre.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:25:34+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel8-Pre.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel8-Pre", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel8-Prod.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel8-Prod.log.rdd.json new file mode 100644 index 00000000..6a4ee109 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel8-Prod.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:25:51+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel8-Prod.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel8-Prod", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel8-QA.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel8-QA.log.rdd.json new file mode 100644 index 00000000..c2576825 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel8-QA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:25:18+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel8-QA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel8-QA", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel8-Test.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel8-Test.log.rdd.json new file mode 100644 index 00000000..2c4248e1 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel8-Test.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:25:00+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel8-Test.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel8-Test", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel9-Pre.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel9-Pre.log.rdd.json new file mode 100644 index 00000000..885d98e4 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel9-Pre.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:27:48+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel9-Pre.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel9-Pre", + "result": "ERROR" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel9-Prod.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel9-Prod.log.rdd.json new file mode 100644 index 00000000..87e1ae86 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel9-Prod.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:28:04+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel9-Prod.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel9-Prod", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel9-QA.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel9-QA.log.rdd.json new file mode 100644 index 00000000..df61dbf6 --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel9-QA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:27:33+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel9-QA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel9-QA", + "result": "FAIL" +} diff --git a/DELME/aaa/13d-ccv-promote-rhel9-Test.log.rdd.json b/DELME/aaa/13d-ccv-promote-rhel9-Test.log.rdd.json new file mode 100644 index 00000000..007f687c --- /dev/null +++ b/DELME/aaa/13d-ccv-promote-rhel9-Test.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:27:17+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/13d-ccv-promote-rhel9-Test.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "13d-ccv-promote-rhel9-Test", + "result": "FAIL" +} diff --git a/DELME/aaa/14-capsync-populate.log.rdd.json b/DELME/aaa/14-capsync-populate.log.rdd.json new file mode 100644 index 00000000..1bd76451 --- /dev/null +++ b/DELME/aaa/14-capsync-populate.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T08:28:20+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/14-capsync-populate.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "14-capsync-populate", + "result": "FAIL" +} diff --git a/DELME/aaa/21-cv-publish-big.log.rdd.json b/DELME/aaa/21-cv-publish-big.log.rdd.json new file mode 100644 index 00000000..47e9604f --- /dev/null +++ b/DELME/aaa/21-cv-publish-big.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:00:17+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/21-cv-publish-big.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "21-cv-publish-big", + "result": "FAIL" +} diff --git a/DELME/aaa/23-cv-promote-big-BenchLifeEnvAAA-BenchLifeEnvBBB.log.rdd.json b/DELME/aaa/23-cv-promote-big-BenchLifeEnvAAA-BenchLifeEnvBBB.log.rdd.json new file mode 100644 index 00000000..7dd5bf76 --- /dev/null +++ b/DELME/aaa/23-cv-promote-big-BenchLifeEnvAAA-BenchLifeEnvBBB.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:01:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/23-cv-promote-big-BenchLifeEnvAAA-BenchLifeEnvBBB.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "23-cv-promote-big-BenchLifeEnvAAA-BenchLifeEnvBBB", + "result": "FAIL" +} diff --git a/DELME/aaa/23-cv-promote-big-BenchLifeEnvBBB-BenchLifeEnvCCC.log.rdd.json b/DELME/aaa/23-cv-promote-big-BenchLifeEnvBBB-BenchLifeEnvCCC.log.rdd.json new file mode 100644 index 00000000..887b3672 --- /dev/null +++ b/DELME/aaa/23-cv-promote-big-BenchLifeEnvBBB-BenchLifeEnvCCC.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:01:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/23-cv-promote-big-BenchLifeEnvBBB-BenchLifeEnvCCC.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "23-cv-promote-big-BenchLifeEnvBBB-BenchLifeEnvCCC", + "result": "FAIL" +} diff --git a/DELME/aaa/23-cv-promote-big-Library-BenchLifeEnvAAA.log.rdd.json b/DELME/aaa/23-cv-promote-big-Library-BenchLifeEnvAAA.log.rdd.json new file mode 100644 index 00000000..d7480f73 --- /dev/null +++ b/DELME/aaa/23-cv-promote-big-Library-BenchLifeEnvAAA.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:00:46+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/23-cv-promote-big-Library-BenchLifeEnvAAA.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "23-cv-promote-big-Library-BenchLifeEnvAAA", + "result": "FAIL" +} diff --git a/DELME/aaa/33-cv-filtered-publish.log.rdd.json b/DELME/aaa/33-cv-filtered-publish.log.rdd.json new file mode 100644 index 00000000..ae67249e --- /dev/null +++ b/DELME/aaa/33-cv-filtered-publish.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:02:03+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/33-cv-filtered-publish.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "33-cv-filtered-publish", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-112-Register.log.rdd.json b/DELME/aaa/48-register-112-Register.log.rdd.json new file mode 100644 index 00000000..aa74b173 --- /dev/null +++ b/DELME/aaa/48-register-112-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:34:18+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-112-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-112-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-168-Register.log.rdd.json b/DELME/aaa/48-register-168-Register.log.rdd.json new file mode 100644 index 00000000..218bb8cd --- /dev/null +++ b/DELME/aaa/48-register-168-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:40:35+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-168-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-168-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-224-Register.log.rdd.json b/DELME/aaa/48-register-224-Register.log.rdd.json new file mode 100644 index 00000000..4ebf66fd --- /dev/null +++ b/DELME/aaa/48-register-224-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:50:25+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-224-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-224-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-280-Register.log.rdd.json b/DELME/aaa/48-register-280-Register.log.rdd.json new file mode 100644 index 00000000..472a3a74 --- /dev/null +++ b/DELME/aaa/48-register-280-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T18:59:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-280-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-280-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-336-Register.log.rdd.json b/DELME/aaa/48-register-336-Register.log.rdd.json new file mode 100644 index 00000000..691b7976 --- /dev/null +++ b/DELME/aaa/48-register-336-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:29:26+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-336-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-336-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-392-Register.log.rdd.json b/DELME/aaa/48-register-392-Register.log.rdd.json new file mode 100644 index 00000000..4dbcc764 --- /dev/null +++ b/DELME/aaa/48-register-392-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:59:01+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-392-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-392-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-448-Register.log.rdd.json b/DELME/aaa/48-register-448-Register.log.rdd.json new file mode 100644 index 00000000..ee11508e --- /dev/null +++ b/DELME/aaa/48-register-448-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T11:32:00+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-448-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-448-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-504-Register.log.rdd.json b/DELME/aaa/48-register-504-Register.log.rdd.json new file mode 100644 index 00000000..fe3ceef4 --- /dev/null +++ b/DELME/aaa/48-register-504-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T12:12:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-504-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-504-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-56-Register.log.rdd.json b/DELME/aaa/48-register-56-Register.log.rdd.json new file mode 100644 index 00000000..0682f123 --- /dev/null +++ b/DELME/aaa/48-register-56-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:29:54+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-56-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-56-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-560-Register.log.rdd.json b/DELME/aaa/48-register-560-Register.log.rdd.json new file mode 100644 index 00000000..bc260a11 --- /dev/null +++ b/DELME/aaa/48-register-560-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T12:57:03+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-560-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-560-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-616-Register.log.rdd.json b/DELME/aaa/48-register-616-Register.log.rdd.json new file mode 100644 index 00000000..478c13d3 --- /dev/null +++ b/DELME/aaa/48-register-616-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T13:44:30+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-616-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-616-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-672-Register.log.rdd.json b/DELME/aaa/48-register-672-Register.log.rdd.json new file mode 100644 index 00000000..8052f00e --- /dev/null +++ b/DELME/aaa/48-register-672-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T14:35:40+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-672-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-672-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-728-Register.log.rdd.json b/DELME/aaa/48-register-728-Register.log.rdd.json new file mode 100644 index 00000000..b575639a --- /dev/null +++ b/DELME/aaa/48-register-728-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T15:32:01+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-728-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-728-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-784-Register.log.rdd.json b/DELME/aaa/48-register-784-Register.log.rdd.json new file mode 100644 index 00000000..efb773b9 --- /dev/null +++ b/DELME/aaa/48-register-784-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T16:36:31+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-784-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-784-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-840-Register.log.rdd.json b/DELME/aaa/48-register-840-Register.log.rdd.json new file mode 100644 index 00000000..fa04c8e0 --- /dev/null +++ b/DELME/aaa/48-register-840-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T17:42:53+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-840-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-840-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/48-register-overall-Register.log.rdd.json b/DELME/aaa/48-register-overall-Register.log.rdd.json new file mode 100644 index 00000000..50eb64d3 --- /dev/null +++ b/DELME/aaa/48-register-overall-Register.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:29:54+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/48-register-overall-Register.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "48-register-overall-Register", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-1176-duration.log.rdd.json b/DELME/aaa/55-rex-date-1176-duration.log.rdd.json new file mode 100644 index 00000000..7bf056be --- /dev/null +++ b/DELME/aaa/55-rex-date-1176-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:37:57+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-1176-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-1176-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-1568-duration.log.rdd.json b/DELME/aaa/55-rex-date-1568-duration.log.rdd.json new file mode 100644 index 00000000..57aee141 --- /dev/null +++ b/DELME/aaa/55-rex-date-1568-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T11:08:43+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-1568-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-1568-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-168-duration.log.rdd.json b/DELME/aaa/55-rex-date-168-duration.log.rdd.json new file mode 100644 index 00000000..5f025c5f --- /dev/null +++ b/DELME/aaa/55-rex-date-168-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:38:00+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-168-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-168-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-2016-duration.log.rdd.json b/DELME/aaa/55-rex-date-2016-duration.log.rdd.json new file mode 100644 index 00000000..7fe684a5 --- /dev/null +++ b/DELME/aaa/55-rex-date-2016-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T11:45:54+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-2016-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-2016-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-2520-duration.log.rdd.json b/DELME/aaa/55-rex-date-2520-duration.log.rdd.json new file mode 100644 index 00000000..7e079b31 --- /dev/null +++ b/DELME/aaa/55-rex-date-2520-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T12:27:14+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-2520-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-2520-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-3080-duration.log.rdd.json b/DELME/aaa/55-rex-date-3080-duration.log.rdd.json new file mode 100644 index 00000000..e6cb88c0 --- /dev/null +++ b/DELME/aaa/55-rex-date-3080-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T13:12:28+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-3080-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-3080-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-336-duration.log.rdd.json b/DELME/aaa/55-rex-date-336-duration.log.rdd.json new file mode 100644 index 00000000..52505f44 --- /dev/null +++ b/DELME/aaa/55-rex-date-336-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:45:46+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-336-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-336-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-3696-duration.log.rdd.json b/DELME/aaa/55-rex-date-3696-duration.log.rdd.json new file mode 100644 index 00000000..05d639b1 --- /dev/null +++ b/DELME/aaa/55-rex-date-3696-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T14:00:20+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-3696-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-3696-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-4368-duration.log.rdd.json b/DELME/aaa/55-rex-date-4368-duration.log.rdd.json new file mode 100644 index 00000000..fe5fb88e --- /dev/null +++ b/DELME/aaa/55-rex-date-4368-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T14:53:19+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-4368-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-4368-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-5096-duration.log.rdd.json b/DELME/aaa/55-rex-date-5096-duration.log.rdd.json new file mode 100644 index 00000000..cdd8621d --- /dev/null +++ b/DELME/aaa/55-rex-date-5096-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T15:53:25+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-5096-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-5096-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-56-duration.log.rdd.json b/DELME/aaa/55-rex-date-56-duration.log.rdd.json new file mode 100644 index 00000000..8342b576 --- /dev/null +++ b/DELME/aaa/55-rex-date-56-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:31:48+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-56-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-56-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-560-duration.log.rdd.json b/DELME/aaa/55-rex-date-560-duration.log.rdd.json new file mode 100644 index 00000000..f7fd9ec4 --- /dev/null +++ b/DELME/aaa/55-rex-date-560-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:57:12+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-560-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-560-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-5880-duration.log.rdd.json b/DELME/aaa/55-rex-date-5880-duration.log.rdd.json new file mode 100644 index 00000000..7f0e829a --- /dev/null +++ b/DELME/aaa/55-rex-date-5880-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T16:56:21+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-5880-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-5880-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-6720-duration.log.rdd.json b/DELME/aaa/55-rex-date-6720-duration.log.rdd.json new file mode 100644 index 00000000..0317e921 --- /dev/null +++ b/DELME/aaa/55-rex-date-6720-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T18:06:07+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-6720-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-6720-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-7000-duration.log.rdd.json b/DELME/aaa/55-rex-date-7000-duration.log.rdd.json new file mode 100644 index 00000000..f437e068 --- /dev/null +++ b/DELME/aaa/55-rex-date-7000-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T19:07:33+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-7000-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-7000-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/55-rex-date-840-duration.log.rdd.json b/DELME/aaa/55-rex-date-840-duration.log.rdd.json new file mode 100644 index 00000000..3a948e9b --- /dev/null +++ b/DELME/aaa/55-rex-date-840-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:10:30+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/55-rex-date-840-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "55-rex-date-840-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-1176-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-1176-duration.log.rdd.json new file mode 100644 index 00000000..ad287a1b --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-1176-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:53:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-1176-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-1176-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-1568-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-1568-duration.log.rdd.json new file mode 100644 index 00000000..bb283426 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-1568-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T11:25:18+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-1568-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-1568-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-168-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-168-duration.log.rdd.json new file mode 100644 index 00000000..c872a5f8 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-168-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:39:15+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-168-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-168-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-2016-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-2016-duration.log.rdd.json new file mode 100644 index 00000000..14f6891c --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-2016-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T12:03:37+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-2016-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-2016-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-2520-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-2520-duration.log.rdd.json new file mode 100644 index 00000000..e3516b75 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-2520-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T12:46:04+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-2520-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-2520-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-3080-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-3080-duration.log.rdd.json new file mode 100644 index 00000000..73a7c72f --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-3080-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T13:31:26+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-3080-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-3080-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-336-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-336-duration.log.rdd.json new file mode 100644 index 00000000..27809b75 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-336-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:48:03+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-336-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-336-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-3696-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-3696-duration.log.rdd.json new file mode 100644 index 00000000..85f565cc --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-3696-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T14:20:26+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-3696-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-3696-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-4368-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-4368-duration.log.rdd.json new file mode 100644 index 00000000..361559df --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-4368-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T15:13:34+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-4368-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-4368-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-5096-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-5096-duration.log.rdd.json new file mode 100644 index 00000000..ab9a3741 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-5096-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T16:14:53+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-5096-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-5096-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-56-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-56-duration.log.rdd.json new file mode 100644 index 00000000..f4fc14b7 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-56-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:33:00+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-56-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-56-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-560-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-560-duration.log.rdd.json new file mode 100644 index 00000000..34583bfe --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-560-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T09:59:31+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-560-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-560-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-5880-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-5880-duration.log.rdd.json new file mode 100644 index 00000000..24d34fe0 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-5880-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T17:19:02+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-5880-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-5880-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-6720-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-6720-duration.log.rdd.json new file mode 100644 index 00000000..803b6f58 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-6720-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T18:32:04+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-6720-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-6720-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-7000-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-7000-duration.log.rdd.json new file mode 100644 index 00000000..3e526e97 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-7000-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T19:34:31+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-7000-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-7000-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/56-rex-date-ansible-840-duration.log.rdd.json b/DELME/aaa/56-rex-date-ansible-840-duration.log.rdd.json new file mode 100644 index 00000000..ae5d6105 --- /dev/null +++ b/DELME/aaa/56-rex-date-ansible-840-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T10:24:56+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/56-rex-date-ansible-840-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "56-rex-date-ansible-840-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/59-rex-katello_package_update-7000-duration.log.rdd.json b/DELME/aaa/59-rex-katello_package_update-7000-duration.log.rdd.json new file mode 100644 index 00000000..f001c2ac --- /dev/null +++ b/DELME/aaa/59-rex-katello_package_update-7000-duration.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T20:03:45+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/59-rex-katello_package_update-7000-duration.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "59-rex-katello_package_update-7000-duration", + "result": "FAIL" +} diff --git a/DELME/aaa/61-hammer-list-HammerHostList.log.rdd.json b/DELME/aaa/61-hammer-list-HammerHostList.log.rdd.json new file mode 100644 index 00000000..1e66b238 --- /dev/null +++ b/DELME/aaa/61-hammer-list-HammerHostList.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T21:49:36+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/61-hammer-list-HammerHostList.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "61-hammer-list-HammerHostList", + "result": "FAIL" +} diff --git a/DELME/aaa/61-hammer-list.log.rdd.json b/DELME/aaa/61-hammer-list.log.rdd.json new file mode 100644 index 00000000..6005de37 --- /dev/null +++ b/DELME/aaa/61-hammer-list.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T21:49:35+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/61-hammer-list.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "61-hammer-list", + "result": "FAIL" +} diff --git a/DELME/aaa/62-webui-pages-WebUIPagesTest_c10_d300.log.rdd.json b/DELME/aaa/62-webui-pages-WebUIPagesTest_c10_d300.log.rdd.json new file mode 100644 index 00000000..8c9fdcb7 --- /dev/null +++ b/DELME/aaa/62-webui-pages-WebUIPagesTest_c10_d300.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T21:53:05+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/62-webui-pages-WebUIPagesTest_c10_d300.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "62-webui-pages-WebUIPagesTest_c10_d300", + "result": "FAIL" +} diff --git a/DELME/aaa/63-foreman_inventory_upload-report-generate.log.rdd.json b/DELME/aaa/63-foreman_inventory_upload-report-generate.log.rdd.json new file mode 100644 index 00000000..f2bec5ff --- /dev/null +++ b/DELME/aaa/63-foreman_inventory_upload-report-generate.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T21:58:12+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/63-foreman_inventory_upload-report-generate.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "63-foreman_inventory_upload-report-generate", + "result": "FAIL" +} diff --git a/DELME/aaa/70-backup-BackupOffline.log.rdd.json b/DELME/aaa/70-backup-BackupOffline.log.rdd.json new file mode 100644 index 00000000..58545da1 --- /dev/null +++ b/DELME/aaa/70-backup-BackupOffline.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:15:49+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/70-backup-BackupOffline.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "70-backup-BackupOffline", + "result": "FAIL" +} diff --git a/DELME/aaa/70-backup-BackupOnline.log.rdd.json b/DELME/aaa/70-backup-BackupOnline.log.rdd.json new file mode 100644 index 00000000..f75a4cf8 --- /dev/null +++ b/DELME/aaa/70-backup-BackupOnline.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T21:58:41+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/70-backup-BackupOnline.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "70-backup-BackupOnline", + "result": "FAIL" +} diff --git a/DELME/aaa/70-backup-RestoreOffline.log.rdd.json b/DELME/aaa/70-backup-RestoreOffline.log.rdd.json new file mode 100644 index 00000000..1082b52b --- /dev/null +++ b/DELME/aaa/70-backup-RestoreOffline.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:21:41+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/70-backup-RestoreOffline.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "70-backup-RestoreOffline", + "result": "FAIL" +} diff --git a/DELME/aaa/70-backup-RestoreOnline.log.rdd.json b/DELME/aaa/70-backup-RestoreOnline.log.rdd.json new file mode 100644 index 00000000..a8a8ddcf --- /dev/null +++ b/DELME/aaa/70-backup-RestoreOnline.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:01:51+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/70-backup-RestoreOnline.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "70-backup-RestoreOnline", + "result": "FAIL" +} diff --git a/DELME/aaa/80-test-sync-repositories-PromoteContentViews.log.rdd.json b/DELME/aaa/80-test-sync-repositories-PromoteContentViews.log.rdd.json new file mode 100644 index 00000000..b4a84541 --- /dev/null +++ b/DELME/aaa/80-test-sync-repositories-PromoteContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:39:04+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/80-test-sync-repositories-PromoteContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "80-test-sync-repositories-PromoteContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/80-test-sync-repositories-PublishContentViews.log.rdd.json b/DELME/aaa/80-test-sync-repositories-PublishContentViews.log.rdd.json new file mode 100644 index 00000000..b48b96ee --- /dev/null +++ b/DELME/aaa/80-test-sync-repositories-PublishContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:38:04+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/80-test-sync-repositories-PublishContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "80-test-sync-repositories-PublishContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/80-test-sync-repositories-SyncRepositories.log.rdd.json b/DELME/aaa/80-test-sync-repositories-SyncRepositories.log.rdd.json new file mode 100644 index 00000000..3a5352b0 --- /dev/null +++ b/DELME/aaa/80-test-sync-repositories-SyncRepositories.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:34:16+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/80-test-sync-repositories-SyncRepositories.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "80-test-sync-repositories-SyncRepositories", + "result": "FAIL" +} diff --git a/DELME/aaa/80-test-sync-repositories.log.rdd.json b/DELME/aaa/80-test-sync-repositories.log.rdd.json new file mode 100644 index 00000000..4cebbf9f --- /dev/null +++ b/DELME/aaa/80-test-sync-repositories.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:32:55+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/80-test-sync-repositories.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "80-test-sync-repositories", + "result": "FAIL" +} diff --git a/DELME/aaa/81-test-sync-iso-PromoteContentViews.log.rdd.json b/DELME/aaa/81-test-sync-iso-PromoteContentViews.log.rdd.json new file mode 100644 index 00000000..7670a144 --- /dev/null +++ b/DELME/aaa/81-test-sync-iso-PromoteContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:44:15+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/81-test-sync-iso-PromoteContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "81-test-sync-iso-PromoteContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/81-test-sync-iso-PublishContentViews.log.rdd.json b/DELME/aaa/81-test-sync-iso-PublishContentViews.log.rdd.json new file mode 100644 index 00000000..aec476ed --- /dev/null +++ b/DELME/aaa/81-test-sync-iso-PublishContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:43:15+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/81-test-sync-iso-PublishContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "81-test-sync-iso-PublishContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/81-test-sync-iso-SyncRepositories.log.rdd.json b/DELME/aaa/81-test-sync-iso-SyncRepositories.log.rdd.json new file mode 100644 index 00000000..19515ad9 --- /dev/null +++ b/DELME/aaa/81-test-sync-iso-SyncRepositories.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:41:25+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/81-test-sync-iso-SyncRepositories.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "81-test-sync-iso-SyncRepositories", + "result": "FAIL" +} diff --git a/DELME/aaa/81-test-sync-iso.log.rdd.json b/DELME/aaa/81-test-sync-iso.log.rdd.json new file mode 100644 index 00000000..054a4a7d --- /dev/null +++ b/DELME/aaa/81-test-sync-iso.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:40:27+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/81-test-sync-iso.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "81-test-sync-iso", + "result": "FAIL" +} diff --git a/DELME/aaa/82-test-sync-docker-PromoteContentViews.log.rdd.json b/DELME/aaa/82-test-sync-docker-PromoteContentViews.log.rdd.json new file mode 100644 index 00000000..3173610c --- /dev/null +++ b/DELME/aaa/82-test-sync-docker-PromoteContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:58:52+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/82-test-sync-docker-PromoteContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "82-test-sync-docker-PromoteContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/82-test-sync-docker-PublishContentViews.log.rdd.json b/DELME/aaa/82-test-sync-docker-PublishContentViews.log.rdd.json new file mode 100644 index 00000000..af24b8e8 --- /dev/null +++ b/DELME/aaa/82-test-sync-docker-PublishContentViews.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:57:50+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/82-test-sync-docker-PublishContentViews.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "82-test-sync-docker-PublishContentViews", + "result": "FAIL" +} diff --git a/DELME/aaa/82-test-sync-docker-SyncRepositories.log.rdd.json b/DELME/aaa/82-test-sync-docker-SyncRepositories.log.rdd.json new file mode 100644 index 00000000..71724730 --- /dev/null +++ b/DELME/aaa/82-test-sync-docker-SyncRepositories.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:46:36+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/82-test-sync-docker-SyncRepositories.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "82-test-sync-docker-SyncRepositories", + "result": "FAIL" +} diff --git a/DELME/aaa/82-test-sync-docker.log.rdd.json b/DELME/aaa/82-test-sync-docker.log.rdd.json new file mode 100644 index 00000000..50242113 --- /dev/null +++ b/DELME/aaa/82-test-sync-docker.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T22:45:38+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/82-test-sync-docker.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "82-test-sync-docker", + "result": "FAIL" +} diff --git a/DELME/aaa/99-remove-hosts-if-any.log.rdd.json b/DELME/aaa/99-remove-hosts-if-any.log.rdd.json new file mode 100644 index 00000000..e46be9ac --- /dev/null +++ b/DELME/aaa/99-remove-hosts-if-any.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T23:00:16+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/99-remove-hosts-if-any.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "99-remove-hosts-if-any", + "result": "FAIL" +} diff --git a/DELME/aaa/capsules-specific.log.rdd.json b/DELME/aaa/capsules-specific.log.rdd.json new file mode 100644 index 00000000..2dbd9ad6 --- /dev/null +++ b/DELME/aaa/capsules-specific.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "stream", + "release": "stream", + "date": "2024-06-09T07:18:41+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/capsules-specific.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "capsules-specific", + "result": "FAIL" +} diff --git a/DELME/aaa/satellite-specific-SatelliteInstallerScenarioSatellite.log.rdd.json b/DELME/aaa/satellite-specific-SatelliteInstallerScenarioSatellite.log.rdd.json new file mode 100644 index 00000000..e6aa76b4 --- /dev/null +++ b/DELME/aaa/satellite-specific-SatelliteInstallerScenarioSatellite.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "stream", + "release": "stream", + "date": "2024-06-09T07:11:35+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/satellite-specific-SatelliteInstallerScenarioSatellite.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "satellite-specific-SatelliteInstallerScenarioSatellite", + "result": "FAIL" +} diff --git a/DELME/aaa/sosreporter-gatherer.log.rdd.json b/DELME/aaa/sosreporter-gatherer.log.rdd.json new file mode 100644 index 00000000..f13f3198 --- /dev/null +++ b/DELME/aaa/sosreporter-gatherer.log.rdd.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.16.0-0.5.stream.el9sat.noarch", + "release": "stream", + "date": "2024-06-09T23:42:37+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerfStreamEL9/run-2024-06-09T06:51:17+00:00/sosreporter-gatherer.log", + "result_id": "run-2024-06-09T06:51:17+00:00", + "test": "sosreporter-gatherer", + "result": "FAIL" +} diff --git a/DELME/aaa2.json b/DELME/aaa2.json new file mode 100644 index 00000000..9c5532f9 --- /dev/null +++ b/DELME/aaa2.json @@ -0,0 +1,11 @@ +{ + "group": "Core Platforms", + "product": "Red Hat Satellite", + "version": "satellite-6.13.7-1.el8sat.noarch", + "release": "6.13", + "date": "2024-06-11T03:00:50+00:00", + "link": "https://workdir-exporter-jenkins-csb-perf.apps.ocp-c1.prod.psi.redhat.com/workspace/ContPerf613EL8/run-2024-06-10T16:46:28+00:00/55-rex-date-5880-duration.log", + "result_id": "run-2024-06-10T16:46:28+00:00", + "test": "55-rex-date-5880-duration", + "result": "FAIL" + } diff --git a/DELME/bbb.py b/DELME/bbb.py new file mode 100644 index 00000000..edaac15d --- /dev/null +++ b/DELME/bbb.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import opl.generators.inventory_ingress +import opl.generators.qpc_tarball + + diff --git a/DELME/ccc.py b/DELME/ccc.py new file mode 100644 index 00000000..a45a4878 --- /dev/null +++ b/DELME/ccc.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +launches_to_check = [ + { + "name": "InsightsCompliance_runner", + "history": 10, + "owner": "jsemjkal", + }, + { + "name": "InsightsEdge_runner", + "history": 10, + "owner": "rajchauh", + }, + { + "name": "InsightsEngine_runner", + "history": 10, + "owner": "rajchauh", + }, + { + "name": "InsightsFrontEnd_runner", + "history": 300, + "owner": "lrios", + }, + { + "name": "InsightsInsightsCore_runner", + "history": 10, + "owner": "spadakan", + }, + { + "name": "InsightsInventory_runner", + "history": 10, + "owner": "spadakan", + }, + { + "name": "InsightsInventoryHBIPerfTest_runner", + "history": 10, + "owner": "spadakan", + }, + { + "name": "InsightsNotifications_runner", + "history": 10, + "owner": "lrios", + }, + { + "name": "InsightsRBAC_runner", + "history": 10, + "owner": "rajchauh", + }, + { + "name": "InsightsTally_runner", + "history": 10, + "owner": "cmusali", + }, + { + "name": "InsightsYuptoo_runner", + "history": 10, + "owner": "shubansa", + }, + ] +import yaml +with open("/tmp/ccc.yaml", "w") as fp: + yaml.dump(launches_to_check, fp) diff --git a/DELME/cluster_read_rhosak-cluster.yaml b/DELME/cluster_read_rhosak-cluster.yaml new file mode 100644 index 00000000..1667b624 --- /dev/null +++ b/DELME/cluster_read_rhosak-cluster.yaml @@ -0,0 +1,44 @@ +{% set namespace=MY_ENV_NAMESPACE %} + +- name: measurements.openshift-monitoring.sum-container_memory_working_set_bytes + monitoring_query: sum(container_memory_working_set_bytes{pod=~'kafka-instance-kafka-[0-9]+',namespace='{{ namespace }}',container=''}) + monitoring_step: 15 + +- name: measurements.openshift-monitoring.sum-pod-container_cpu_usage-sum + monitoring_query: sum(pod:container_cpu_usage:sum{pod=~'kafka-instance-kafka-[0-9]+',namespace='{{ namespace }}'}) + monitoring_step: 15 + +- name: measurements.openshift-monitoring.sum-irate-container_network_receive_bytes_total + monitoring_query: sum(irate(container_network_receive_bytes_total{pod=~'kafka-instance-kafka-[0-9]+', namespace='{{ namespace }}'}[5m])) + monitoring_step: 15 + +- name: measurements.openshift-monitoring.sum-irate-container_network_transmit_bytes_total + monitoring_query: sum(irate(container_network_transmit_bytes_total{pod=~'kafka-instance-kafka-[0-9]+', namespace='{{ namespace }}'}[5m])) + monitoring_step: 15 + +- name: measurements.openshift-monitoring.sum-pod-container_fs_usage_bytes-sum + monitoring_query: sum(pod:container_fs_usage_bytes:sum{pod=~'kafka-instance-kafka-[0-9]+',namespace='{{ namespace }}'}) + monitoring_step: 15 + +- name: measurements.openshift-monitoring.sum-kube_pod_container_status_restarts_total + monitoring_query: sum(kube_pod_container_status_restarts_total{pod=~'kafka-instance-kafka-[0-9]+',namespace='{{ namespace }}'}) + monitoring_step: 15 + +###- name: parameters.cluster.pods.upload-service.count +### command: oc -n qa get pods -l app=upload-service -o name | wc -l +###- name: parameters.cluster.pods.insights-puptoo.count +### command: oc -n qa get pods -l app=insights-puptoo -o name | wc -l +###- name: parameters.cluster.pods.insights-storage-broker.count +### command: oc -n qa get pods -l app=insights-storage-broker -o name | wc -l +### +#### TODO: We should query deployment config here I think +###- name: parameters.cluster.pods.vulnerability-engine-listener.resources +### command: oc -n qa get pods -l deploymentconfig=vulnerability-engine-listener -o json | python3 -c "import sys, json; print(json.dumps(json.load(sys.stdin)['items'][0]['spec']['containers'][0]['resources']))" +### output: json +### +###- name: measurements.insights-inventory-mq-service.cpu +### monitoring_query: sum(pod_name:container_cpu_usage:sum{pod_name=~'insights-inventory-mq-service-.*',namespace='qa'}) +### monitoring_step: 60 +### +###- name: measurements.satellite.load +### grafana_target: $Cloud.$Node.load.load.shortterm diff --git a/DELME/cluster_read_rhosak-kafka.yaml b/DELME/cluster_read_rhosak-kafka.yaml new file mode 100644 index 00000000..92044a52 --- /dev/null +++ b/DELME/cluster_read_rhosak-kafka.yaml @@ -0,0 +1,57 @@ +{% set namespace=MY_ENV_NAMESPACE %} + +- name: measurements.managed-application-services-observability.sum-haproxy_server_bytes_in_total + monitoring_query: sum(haproxy_server_bytes_in_total{exported_pod=~"kafka-instance-kafka-[0-9]+",exported_namespace="{{ namespace }}"}) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-haproxy_server_bytes_out_total + monitoring_query: sum(haproxy_server_bytes_out_total{exported_pod=~"kafka-instance-kafka-[0-9]+",exported_namespace="{{ namespace }}"}) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-kafka_broker_quota_totalstorageusedbytes + monitoring_query: sum(kafka_broker_quota_totalstorageusedbytes{namespace!=""}) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-kafka_topic_partition_current_offset + monitoring_query: sum(rate(kafka_topic_partition_current_offset[5m])) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-kubelet_volume_stats_available_bytes + monitoring_query: sum(rate(kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data-([0-9]+)?-(.+)-kafka-[0-9]+"}[5m])) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-kafka_server_replicamanager_under_replicated_partitions + monitoring_query: sum(kafka_server_replicamanager_under_replicated_partitions) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-kafka_controller_kafkacontroller_offline_partitions_count + monitoring_query: sum(kafka_controller_kafkacontroller_offline_partitions_count) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-kafka_cluster_partition_under_min_isr + monitoring_query: sum(kafka_cluster_partition_under_min_isr) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-kube_pod_container_status_restarts_total + monitoring_query: sum(rate(kube_pod_container_status_restarts_total{container="kafka"}[5m])) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-ALERTS + monitoring_query: sum(rate(ALERTS{alertstate="firing"}[5m])) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-zookeeper_avg_request_latency + monitoring_query: sum(zookeeper_avg_request_latency) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-zookeeper_outstanding_requests + monitoring_query: sum(zookeeper_outstanding_requests) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-kube_pod_container_status_restarts_total + monitoring_query: sum(rate(kube_pod_container_status_restarts_total{container="zookeeper"}[5m])) + monitoring_step: 15 + +- name: measurements.managed-application-services-observability.sum-rate-kafka_log_log_size + monitoring_query: sum(rate(kafka_log_log_size{namespace=~"kafka-.*",strimzi_io_kind=~"Kafka"}[5m])) + monitoring_step: 15 diff --git a/DELME/data.txt b/DELME/data.txt new file mode 100644 index 00000000..01f02e32 --- /dev/null +++ b/DELME/data.txt @@ -0,0 +1 @@ +bbb \ No newline at end of file diff --git a/DELME/ddd.py b/DELME/ddd.py new file mode 100644 index 00000000..c7bad5ac --- /dev/null +++ b/DELME/ddd.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import argparse +import sys +import os +import logging +from locust import HttpUser +from locust import constant +from locust import task +import opl.args +import opl.gen +import opl.locust +import opl.skelet +import base64 +import json +import size_kb + + +class TheNotifications(HttpUser): + + wait_time = constant( + 0 + ) # IF performing latency test, then change to 1, but not necessary here + + def getting_headers(self): + """ + This header was created to avoid issues with WARNING when sending emails + """ + data = { + "identity": { + "associate": { + "Role": ["org_admin"], + "email": "tester@example.com", + "givenName": "tester", + "rhatUUID": "01234567-89ab-cdef-0123-456789abcdef", + "surname": "tester", + }, + "auth_type": "saml-auth", + "type": "Associate", + } + } + + return base64.b64encode(json.dumps(data).encode("UTF-8")) + + def _post_availability(self, endpoint, size_payload): + """Just a helper for a given endpoint with given limit set""" + + url = f"{self.host_base}/{endpoint}" + + headers = { + "Content-Type": "application/json", + "x-rh-identity": self.getting_headers(), + } + + payload = json.dumps( + { + "version": "2.0.0", + "bundle": "rhel", + "application": "policies", + "event_type": "policy-triggered", + "timestamp": "2024-05-29T09:08:31.911475014", + "account_id": "12345", + "org_id": "12345", + "context": { + "inventory_id": "93f5896e-91dd-4b9d-a9c4-21ff618ee991", + "system_check_in": "2024-05-29T09:08:31.860645", + "display_name": "gduval-rhel91", + "tags": [ + { + "value": "93f5896e-91dd-4b9d-a9c4-21ff618ee991", + "key": "inventory_id", + }, + {"value": "gduval-rhel91", "key": "display_name"}, + ], + }, + "events": [ + { + "metadata": {}, + "payload": size_kb.size_of_dict(size=size_payload), + } + ], + "recipients": [], + } + ) + + response = self.client.post( + url=url, headers=headers, data=payload, verify=False + ) + return response + + @task(1) + def check_availability(self): + """ + first getting sources id and supplying with check_availability + """ + + return self._post_availability("/notifications", self.size_payload) + + +def doit(args, status_data): + + logging.info(f"Running with test data total counts: {status_data}") + # Extra args + test_set = TheNotifications + + test_set.host_base = f"{args.gw_host}" + args.host = args.gw_host + + test_set.size_payload = int(args.size_payload) + + # Add parameters to status data file. + status_data.set("name", "Insights NOTIFICATIONS API perf test") + status_data.set("parameters.locust.hatch_rate", args.hatch_rate) + status_data.set("parameters.locust.host", args.host) + status_data.set("parameters.locust.num_clients", args.num_clients) + status_data.set("parameters.locust.stop_timeout", args.stop_timeout) + status_data.set("parameters.test.duration", args.test_duration) + status_data.set("parameters.locust.test-requests", args.test_requests) + print( + f"Running with host = {args.gw_host}, num_clients = {args.num_clients}, hatch_rate = {args.hatch_rate} and duration = {args.test_duration} seconds and payload_size(kb) {args.size_payload} and num_requests = {args.test_requests}" + ) + + return opl.locust.run_locust( + args, status_data, test_set, new_stats=True + ) # TODO look into tweeking this + + +def main(): + + print("***** Executing Perf testing ******") + parser = argparse.ArgumentParser( + description="Measure NOTIFICATIONS API endpoints speed", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--gw_host", + dest="gw_host", + default=os.getenv( + "notifications_gw_host", + "http://notifications-gw-service.notifications-perf.svc.cluster.local:8000", + ), # This is Golang + help="GW host, use env variable GW_HOST)", + ) + + parser.add_argument( + "--size_payload", + dest="size_payload", + default="40", + help="Payload size in kb", + ) + + opl.args.add_locust_opts(parser) + with opl.skelet.test_setup(parser) as (args, status_data): + logging.info(f"The status data file is {args.status_data_file}") + + return doit(args, status_data) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/DELME/edge-test-data.json b/DELME/edge-test-data.json new file mode 100644 index 00000000..030417d9 --- /dev/null +++ b/DELME/edge-test-data.json @@ -0,0 +1 @@ +[{"account": "3019212", "image_set_id": [11860], "repo_ids": [11849], "commit_ids": [11846], "installer_ids": [11845], "image_ids": [11841], "os_tree_commits": ["evb68xc6d2b2ns7j6lloueg66dbff4mpnsmc1z83hmvnu1jbx8fppkdm2i2vbqfc"]}] diff --git a/DELME/investigator-conf.yaml b/DELME/investigator-conf.yaml new file mode 100644 index 00000000..03ec491e --- /dev/null +++ b/DELME/investigator-conf.yaml @@ -0,0 +1,107 @@ +# Get previous test results from here +history: + # type: csv + # file: /tmp/data.csv + type: sd_dir + dir: testing_sd_dir/ + matchers: | + name: "{{ current.get('name') }}" + +# Get results of test run to be compared to historical results +current: + type: status_data + file: /tmp/status-data.json + +methods: + - check_by_stdev_1 + - check_by_stdev_2 + - check_by_stdev_3 + - check_by_trim_stdev_1 + - check_by_trim_stdev_2 + - check_by_error_1 + - check_by_error_2 + - check_by_error_3 + - check_by_error_4 + - check_by_error_5 + - check_by_perc_20 + - check_by_perc_40 + - check_by_perc_60 + - check_by_perc_80 + - check_by_perc_100 + - check_by_min_max_0_1 + - check_by_min_max_7_1 + - check_by_min_max_7_2 + - check_by_min_max_7_3 + +# columns in case of CSV input +# JSON paths in case of status_data input +sets: | + {% if current.get('parameters.cli').startswith('experiment/reg-average.py ') %} + - results.items.avg_duration + - results.items.passed + - results.items.report_rc + {% else %} + - results.duration + {% endif %} + + {% macro metrics(name) -%} + {% if current.get(name + '.samples') > 0 -%} + - {{ name }}.mean + {%- endif %} + {%- endmacro %} + + # System wide metrics + {{ metrics('measurements.satellite.disk_octets.read') }} + {{ metrics('measurements.satellite.disk_octets.write') }} + {{ metrics('measurements.satellite.eth1.if_octets.rx') }} + {{ metrics('measurements.satellite.eth1.if_octets.tx') }} + {{ metrics('measurements.satellite.load.load.shortterm') }} + {{ metrics('measurements.satellite.memory.memory-used') }} + {{ metrics('measurements.satellite.swap.swap-used') }} + + # Satellite services + {{ metrics('measurements.satellite.processes-httpd.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-httpd.ps_rss') }} + {{ metrics('measurements.satellite.processes-ruby.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-ruby.ps_rss') }} + {{ metrics('measurements.satellite.processes-dynflow.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-dynflow.ps_rss') }} + {{ metrics('measurements.satellite.processes-pulp.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-pulp.ps_rss') }} + {{ metrics('measurements.satellite.processes-qpidd.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-qpidd.ps_rss') }} + {{ metrics('measurements.satellite.processes-qdrouterd.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-qdrouterd.ps_rss') }} + + # Satellite application server services + {{ metrics('measurements.satellite.processes-puma.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-puma.ps_rss') }} + {{ metrics('measurements.satellite.processes-Puma-Worker.ps_count.processes') }} + {{ metrics('measurements.satellite.processes-Puma-Worker.ps_count.threads') }} + {{ metrics('measurements.satellite.processes-Tomcat.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-Tomcat.ps_rss') }} + + # Satellite data store services + {{ metrics('measurements.satellite.processes-postgres.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-postgres.ps_rss') }} + {{ metrics('measurements.satellite.processes-redis.ps_cputime.user') }} + {{ metrics('measurements.satellite.processes-redis.ps_rss') }} + + # PostgreSQL activity metrics + {{ metrics('measurements.satellite.postgresql-candlepin.pg_n_tup_c-del') }} + {{ metrics('measurements.satellite.postgresql-candlepin.pg_n_tup_c-ins') }} + {{ metrics('measurements.satellite.postgresql-candlepin.pg_n_tup_c-upd') }} + {{ metrics('measurements.satellite.postgresql-foreman.pg_n_tup_c-del') }} + {{ metrics('measurements.satellite.postgresql-foreman.pg_n_tup_c-ins') }} + {{ metrics('measurements.satellite.postgresql-foreman.pg_n_tup_c-upd') }} + {{ metrics('measurements.satellite.postgresql-pulpcore.pg_n_tup_c-del') }} + {{ metrics('measurements.satellite.postgresql-pulpcore.pg_n_tup_c-ins') }} + {{ metrics('measurements.satellite.postgresql-pulpcore.pg_n_tup_c-upd') }} + +# Shuld we log our decisions somewhere? +decisions: + ###type: elasticsearch + ###es_server: http://elasticsearch.intlab.perf-infra.lab.eng.rdu2.redhat.com + ###es_index: satellite_perf_aa_decisions + type: csv + filename: /tmp/decisions.csv diff --git a/DELME/load-test.json b/DELME/load-test.json new file mode 100644 index 00000000..42c167dd --- /dev/null +++ b/DELME/load-test.json @@ -0,0 +1,2498 @@ +{ + "ended": "2025-02-27T07:30:24,094835348+00:00", + "measurements": { + "cluster_cpu_usage_seconds_total_rate": { + "iqr": 18.526048059296357, + "max": 39.323793386428946, + "mean": 29.966002813247133, + "median": 30.155535900314398, + "min": 20.229146065930788, + "non_zero_mean": 29.966002813247133, + "non_zero_median": 30.155535900314398, + "percentile25": 20.79774532713259, + "percentile75": 39.323793386428946, + "percentile90": 39.323793386428946, + "percentile99": 39.323793386428946, + "percentile999": 39.323793386428946, + "range": 19.094647320498158, + "samples": 4, + "stdev": 10.809877581404697, + "sum": 119.86401125298853 + }, + "cluster_disk_throughput_total": { + "iqr": 44810311.19561881, + "max": 490676442.2297099, + "mean": 455390087.4233774, + "median": 452128772.41150093, + "min": 426626362.64079785, + "non_zero_mean": 455390087.4233774, + "non_zero_median": 452128772.41150093, + "percentile25": 431354274.3196297, + "percentile75": 476164585.51524854, + "percentile90": 484871699.5439254, + "percentile99": 490095967.9611315, + "percentile999": 490618394.8028521, + "range": 64050079.58891207, + "samples": 4, + "stdev": 30718712.912021983, + "sum": 1821560349.6935096 + }, + "cluster_memory_usage_rss_total": { + "iqr": 1368529920.0, + "max": 235010789376.0, + "mean": 233554392064.0, + "median": 233285949440.0, + "min": 232634880000.0, + "non_zero_mean": 233554392064.0, + "non_zero_median": 233285949440.0, + "percentile25": 232735905792.0, + "percentile75": 234104435712.0, + "percentile90": 234648247910.4, + "percentile99": 234974535229.43997, + "percentile999": 235007163961.344, + "range": 2375909376.0, + "samples": 4, + "stdev": 1102117819.925486, + "sum": 934217568256.0 + }, + "cluster_network_bytes_total": { + "iqr": 1618902013.9201927, + "max": 9084788499.28534, + "mean": 7868590944.7379875, + "median": 7922908802.28728, + "min": 6543757675.092052, + "non_zero_mean": 7868590944.7379875, + "non_zero_median": 7922908802.28728, + "percentile25": 7086298866.552538, + "percentile75": 8705200880.47273, + "percentile90": 8932953451.760296, + "percentile99": 9069604994.532835, + "percentile999": 9083270148.81009, + "range": 2541030824.193287, + "samples": 4, + "stdev": 1169083748.312969, + "sum": 31474363778.95195 + }, + "cluster_network_receive_bytes_total": { + "iqr": 427799857.6018348, + "max": 4457126369.928403, + "mean": 3989485463.325246, + "median": 3983181614.5881577, + "min": 3534452254.1962643, + "non_zero_mean": 3989485463.325246, + "non_zero_median": 3983181614.5881577, + "percentile25": 3772433610.155784, + "percentile75": 4200233467.757619, + "percentile90": 4354369209.06009, + "percentile99": 4446850653.841572, + "percentile999": 4456098798.319719, + "range": 922674115.7321386, + "samples": 4, + "stdev": 391733589.0604828, + "sum": 15957941853.300983 + }, + "cluster_network_transmit_bytes_total": { + "iqr": 1535740405.6379743, + "max": 4726910945.392903, + "mean": 3879105481.4127426, + "median": 3890102779.681139, + "min": 3009305420.8957887, + "non_zero_mean": 3879105481.4127426, + "non_zero_median": 3890102779.681139, + "percentile25": 3116733927.7279534, + "percentile75": 4652474333.365928, + "percentile90": 4697136300.582113, + "percentile99": 4723933480.911824, + "percentile999": 4726613198.944796, + "range": 1717605524.4971147, + "samples": 4, + "stdev": 924401738.0188278, + "sum": 15516421925.65097 + }, + "cluster_nodes_worker_count": { + "iqr": 0.0, + "max": 17.0, + "mean": 17.0, + "median": 17.0, + "min": 17.0, + "non_zero_mean": 17.0, + "non_zero_median": 17.0, + "percentile25": 17.0, + "percentile75": 17.0, + "percentile90": 17.0, + "percentile99": 17.0, + "percentile999": 17.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 68.0 + }, + "cluster_pods_count": { + "iqr": 3.75, + "max": 966.0, + "mean": 959.25, + "median": 960.0, + "min": 951.0, + "non_zero_mean": 959.25, + "non_zero_median": 960.0, + "percentile25": 957.75, + "percentile75": 961.5, + "percentile90": 964.2, + "percentile99": 965.82, + "percentile999": 965.982, + "range": 15.0, + "samples": 4, + "stdev": 6.18465843842649, + "sum": 3837.0 + }, + "cluster_running_pods_on_workers_count": { + "iqr": 1.25, + "max": 487.0, + "mean": 484.25, + "median": 484.0, + "min": 482.0, + "non_zero_mean": 484.25, + "non_zero_median": 484.0, + "percentile25": 483.5, + "percentile75": 484.75, + "percentile90": 486.1, + "percentile99": 486.91, + "percentile999": 486.991, + "range": 5.0, + "samples": 4, + "stdev": 2.0615528128088303, + "sum": 1937.0 + }, + "etcd": { + "count_ready": { + "iqr": 0.0, + "max": 3.0, + "mean": 3.0, + "median": 3.0, + "min": 3.0, + "non_zero_mean": 3.0, + "non_zero_median": 3.0, + "percentile25": 3.0, + "percentile75": 3.0, + "percentile90": 3.0, + "percentile99": 3.0, + "percentile999": 3.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 12.0 + }, + "cpu": { + "iqr": 0.00911732903587692, + "max": 0.9116626988591815, + "mean": 0.8921533392381553, + "median": 0.8908786376888831, + "min": 0.8751933827156736, + "non_zero_mean": 0.8921533392381553, + "non_zero_median": 0.8908786376888831, + "percentile25": 0.8869573239455808, + "percentile75": 0.8960746529814577, + "percentile90": 0.905427480508092, + "percentile99": 0.9110391770240726, + "percentile999": 0.9116003466756706, + "range": 0.0364693161435079, + "samples": 4, + "stdev": 0.014961115891385811, + "sum": 3.5686133569526213 + }, + "disk_throughput": { + "iqr": 0.0, + "max": 5367422.741478259, + "mean": 5367422.741478259, + "median": 5367422.741478259, + "min": 5367422.741478259, + "non_zero_mean": 5367422.741478259, + "non_zero_median": 5367422.741478259, + "percentile25": 5367422.741478259, + "percentile75": 5367422.741478259, + "percentile90": 5367422.741478259, + "percentile99": 5367422.741478259, + "percentile999": 5367422.741478259, + "range": 0.0, + "samples": 1, + "stdev": 0.0, + "sum": 5367422.741478259 + }, + "memory": { + "iqr": 34563072.0, + "max": 14788628480.0, + "mean": 14703219712.0, + "median": 14676099072.0, + "min": 14672052224.0, + "non_zero_mean": 14703219712.0, + "non_zero_median": 14676099072.0, + "percentile25": 14672377856.0, + "percentile75": 14706940928.0, + "percentile90": 14755953459.199999, + "percentile99": 14785360977.92, + "percentile999": 14788301729.792, + "range": 116576256.0, + "samples": 4, + "stdev": 57047439.2807902, + "sum": 58812878848.0 + }, + "network_drop": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 1, + "stdev": 0.0, + "sum": 0.0 + }, + "network_throughput": { + "iqr": 0.0, + "max": 65349327.32221075, + "mean": 65349327.32221075, + "median": 65349327.32221075, + "min": 65349327.32221075, + "non_zero_mean": 65349327.32221075, + "non_zero_median": 65349327.32221075, + "percentile25": 65349327.32221075, + "percentile75": 65349327.32221075, + "percentile90": 65349327.32221075, + "percentile99": 65349327.32221075, + "percentile999": 65349327.32221075, + "range": 0.0, + "samples": 1, + "stdev": 0.0, + "sum": 65349327.32221075 + }, + "restarts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 0.0 + } + }, + "etcd_mvcc_db_total_size_in_bytes_average": { + "iqr": 0.0, + "max": 1629193557.3333333, + "mean": 1629193557.3333333, + "median": 1629193557.3333333, + "min": 1629193557.3333333, + "non_zero_mean": 1629193557.3333333, + "non_zero_median": 1629193557.3333333, + "percentile25": 1629193557.3333333, + "percentile75": 1629193557.3333333, + "percentile90": 1629193557.3333335, + "percentile99": 1629193557.3333333, + "percentile999": 1629193557.3333333, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 6516774229.333333 + }, + "etcd_mvcc_db_total_size_in_use_in_bytes_average": { + "iqr": 13713408.0, + "max": 1112455850.6666667, + "mean": 1105599146.6666667, + "median": 1105599146.6666667, + "min": 1098742442.6666667, + "non_zero_mean": 1105599146.6666667, + "non_zero_median": 1105599146.6666667, + "percentile25": 1098742442.6666667, + "percentile75": 1112455850.6666667, + "percentile90": 1112455850.6666667, + "percentile99": 1112455850.6666667, + "percentile999": 1112455850.6666667, + "range": 13713408.0, + "samples": 4, + "stdev": 7917439.800307168, + "sum": 4422396586.666667 + }, + "etcd_request_duration_seconds_average": { + "iqr": 4.799388699723312e-05, + "max": 0.0031187285195139504, + "mean": 0.003086745814942073, + "median": 0.0030837459928309417, + "min": 0.0030607627545924587, + "non_zero_mean": 0.003086745814942073, + "non_zero_median": 0.0030837459928309417, + "percentile25": 0.0030612489603878907, + "percentile75": 0.003109242847385124, + "percentile90": 0.0031149342506624194, + "percentile99": 0.0031183490926287973, + "percentile999": 0.0031186905768254354, + "range": 5.79657649214917e-05, + "samples": 4, + "stdev": 3.0076080762722086e-05, + "sum": 0.012346983259768293 + }, + "etcd_server_quota_backend_bytes_average": { + "iqr": 0.0, + "max": 8589934592.0, + "mean": 8589934592.0, + "median": 8589934592.0, + "min": 8589934592.0, + "non_zero_mean": 8589934592.0, + "non_zero_median": 8589934592.0, + "percentile25": 8589934592.0, + "percentile75": 8589934592.0, + "percentile90": 8589934592.0, + "percentile99": 8589934592.0, + "percentile999": 8589934592.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 34359738368.0 + }, + "node_disk_io_time_seconds_total": { + "iqr": 0.4628139594216807, + "max": 1.5745747506191552, + "mean": 1.3339524578699251, + "median": 1.348358083956244, + "min": 1.0645189129480577, + "non_zero_mean": 1.3339524578699251, + "non_zero_median": 1.348358083956244, + "percentile25": 1.1097482912022443, + "percentile75": 1.572562250623925, + "percentile90": 1.5737697506210633, + "percentile99": 1.574494250619346, + "percentile999": 1.5745667006191744, + "range": 0.5100558376710975, + "samples": 4, + "stdev": 0.2773943458012369, + "sum": 5.3358098314797004 + }, + "pipelinerun_duration_scheduled_seconds": { + "iqr": 0.02094327800659812, + "max": 3138.01825133388, + "mean": 3137.9554214998607, + "median": 3137.9344782218545, + "min": 3137.9344782218545, + "non_zero_mean": 3137.9554214998607, + "non_zero_median": 3137.9344782218545, + "percentile25": 3137.9344782218545, + "percentile75": 3137.955421499861, + "percentile90": 3137.993119400272, + "percentile99": 3138.0157381405193, + "percentile999": 3138.018000014544, + "range": 0.08377311202548299, + "samples": 4, + "stdev": 0.041886556012741494, + "sum": 12551.821685999443 + }, + "scheduler_pending_pods_count": { + "iqr": 1.0, + "max": 1.0, + "mean": 0.5, + "median": 0.5, + "min": 0.0, + "non_zero_mean": 1.0, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 1.0, + "percentile90": 1.0, + "percentile99": 1.0, + "percentile999": 1.0, + "range": 1.0, + "samples": 4, + "stdev": 0.5773502691896257, + "sum": 2.0 + }, + "storage_count_attachable_volumes_in_use": { + "iqr": 6.0, + "max": 27.0, + "mean": 24.0, + "median": 24.0, + "min": 21.0, + "non_zero_mean": 24.0, + "non_zero_median": 24.0, + "percentile25": 21.0, + "percentile75": 27.0, + "percentile90": 27.0, + "percentile99": 27.0, + "percentile999": 27.0, + "range": 6.0, + "samples": 4, + "stdev": 3.4641016151377544, + "sum": 96.0 + }, + "tekton-results-watcher": { + "container[watcher]": { + "memory": { + "iqr": 6598656.0, + "max": 1118412800.0, + "mean": 1115993292.8, + "median": 1118412800.0, + "min": 1111814144.0, + "non_zero_mean": 1115993292.8, + "non_zero_median": 1118412800.0, + "percentile25": 1111814144.0, + "percentile75": 1118412800.0, + "percentile90": 1118412800.0, + "percentile99": 1118412800.0, + "percentile999": 1118412800.0, + "range": 6598656.0, + "samples": 60, + "stdev": 3206690.1376317036, + "sum": 66959597568.0 + } + }, + "count_ready": { + "iqr": 0.0, + "max": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 1.0, + "non_zero_mean": 1.0, + "non_zero_median": 1.0, + "percentile25": 1.0, + "percentile75": 1.0, + "percentile90": 1.0, + "percentile99": 1.0, + "percentile999": 1.0, + "range": 0.0, + "samples": 60, + "stdev": 0.0, + "sum": 60.0 + }, + "cpu": { + "iqr": 0.0034482725182275464, + "max": 0.13844587976971726, + "mean": 0.13332327011369005, + "median": 0.1319638807549247, + "min": 0.1309194391751936, + "non_zero_mean": 0.13332327011369005, + "non_zero_median": 0.1319638807549247, + "percentile25": 0.1309194391751936, + "percentile75": 0.13436771169342115, + "percentile90": 0.13844587976971726, + "percentile99": 0.13844587976971726, + "percentile999": 0.13844587976971728, + "range": 0.007526440594523648, + "samples": 60, + "stdev": 0.0031040075853352497, + "sum": 7.9993962068214035 + }, + "memory": { + "iqr": 6598656.0, + "max": 1143255040.0, + "mean": 1140835532.8, + "median": 1143255040.0, + "min": 1136656384.0, + "non_zero_mean": 1140835532.8, + "non_zero_median": 1143255040.0, + "percentile25": 1136656384.0, + "percentile75": 1143255040.0, + "percentile90": 1143255040.0, + "percentile99": 1143255040.0, + "percentile999": 1143255040.0, + "range": 6598656.0, + "samples": 60, + "stdev": 3206690.1376317036, + "sum": 68450131968.0 + }, + "restarts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 60, + "stdev": 0.0, + "sum": 0.0 + } + }, + "tekton_pipelines_controller_running_pipelineruns_count": { + "iqr": 1.75, + "max": 48.0, + "mean": 45.75, + "median": 45.5, + "min": 44.0, + "non_zero_mean": 45.75, + "non_zero_median": 45.5, + "percentile25": 44.75, + "percentile75": 46.5, + "percentile90": 47.4, + "percentile99": 47.94, + "percentile999": 47.994, + "range": 4.0, + "samples": 4, + "stdev": 1.707825127659933, + "sum": 183.0 + }, + "tekton_pipelines_controller_running_taskruns_throttled_by_node": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 0.0 + }, + "tekton_pipelines_controller_running_taskruns_throttled_by_quota": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 4, + "stdev": 0.0, + "sum": 0.0 + }, + "tekton_tekton_pipelines_controller_workqueue_depth": { + "iqr": 0.5, + "max": 2.0, + "mean": 0.5, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.0, + "non_zero_median": 2.0, + "percentile25": 0.0, + "percentile75": 0.5, + "percentile90": 1.4000000000000004, + "percentile99": 1.9400000000000004, + "percentile999": 1.9940000000000007, + "range": 2.0, + "samples": 4, + "stdev": 1.0, + "sum": 2.0 + } + }, + "metadata": { + "env": { + "BUILD_ID": "38", + "BUILD_TAG": "jenkins-StoneSoupLoadTestProbe-38", + "BUILD_URL": "https://jenkins-csb-perf-master.dno.corp.redhat.com/job/StoneSoupLoadTestProbe/38/", + "HOSTNAME": "cpt-v2-2tr0c", + "JOB_NAME": "StoneSoupLoadTestProbe", + "MEMBER_CLUSTER": "https://api.stone-prod-p02.hjvn.p1.openshiftapps.com:6443/", + "OPENSHIFT_API": null, + "PROW_JOB_ID": null, + "PULL_BASE_REF": null, + "PULL_BASE_SHA": null, + "PULL_HEAD_REF": null, + "PULL_NUMBER": null, + "PULL_PULL_SHA": null, + "PULL_REFS": null, + "REPO_NAME": null, + "REPO_OWNER": null, + "SCENARIO": null + }, + "git": { + "redhat_appstudio": { + "e2e_tests": { + "commit": { + "abbreviated_hash": "f6270a4", + "author_date": "2025-02-27T08:14:09+01:00", + "author_email": "jhutar@redhat.com", + "author_name": "Jan Hutar", + "committer_date": "2025-02-27T08:14:09+01:00", + "hash": "f6270a4dfa71bb332409a054533abf627924e0d6", + "subject": "feat: Add Jenkins specific variables" + } + } + } + }, + "scenario": {} + }, + "name": "Konflux loadtest", + "owner": null, + "parameters": { + "options": { + "ApplicationsCount": 1, + "BuildPipelineSelectorBundle": "", + "ComponentContainerContext": "/", + "ComponentContainerFile": "Dockerfile", + "ComponentRepoRevision": "main", + "ComponentRepoUrl": "https://gitlab.cee.redhat.com/jhutar/nodejs-devfile-sample3", + "ComponentsCount": 1, + "Concurrency": 1, + "FailFast": false, + "JourneyDuration": "1h", + "JourneyRepeats": 1, + "JourneyUntil": "2025-02-27T08:29:38.807156838Z", + "LogDebug": false, + "LogInfo": true, + "LogTrace": false, + "OutputDir": ".", + "PipelineMintmakerDisabled": true, + "PipelineRepoTemplating": false, + "Purge": true, + "PurgeOnly": false, + "QuayRepo": "redhat-user-workloads-stage", + "Stage": true, + "TestScenarioGitURL": "https://github.com/konflux-ci/integration-examples.git", + "TestScenarioPathInRepo": "pipelines/integration_resolver_pipeline_pass.yaml", + "TestScenarioRevision": "main", + "UsernamePrefix": "undef", + "WaitIntegrationTestsPipelines": true, + "WaitPipelines": true + } + }, + "result": null, + "results": { + "durations": { + "errors": { + "pipelineruns": 0, + "steps": 0, + "taskruns": 116, + "total": 116 + }, + "stats": { + "pipelineruns": { + "build": { + "failed": { + "duration": { + "max": 165, + "mean": 137.5, + "min": 110, + "samples": 2, + "stdev": 38.890872965260115 + }, + "idle": { + "max": 165, + "mean": 137.5, + "min": 110, + "samples": 2, + "stdev": 38.890872965260115 + }, + "running": { + "max": 165, + "mean": 137.5, + "min": 110, + "samples": 2, + "stdev": 38.890872965260115 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 2, + "stdev": 0.0 + } + }, + "passed": { + "duration": { + "max": 384, + "mean": 231.76923076923077, + "min": 160, + "samples": 13, + "stdev": 65.40406950406303 + }, + "idle": { + "max": 384, + "mean": 123.61538461538461, + "min": 9, + "samples": 13, + "stdev": 143.5830412813078 + }, + "running": { + "max": 383, + "mean": 231.46153846153845, + "min": 160, + "samples": 13, + "stdev": 65.38299394671292 + }, + "scheduled": { + "max": 1, + "mean": 0.3076923076923077, + "min": 0, + "samples": 13, + "stdev": 0.4803844614152614 + } + } + }, + "test": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 25, + "mean": 10, + "min": 5, + "samples": 8, + "stdev": 6.502746672423453 + }, + "idle": { + "max": 1, + "mean": 0.25, + "min": 0, + "samples": 8, + "stdev": 0.4629100498862757 + }, + "running": { + "max": 25, + "mean": 10, + "min": 5, + "samples": 8, + "stdev": 6.502746672423453 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + } + }, + "steps": { + "build/apply-tags/step-apply-additional-tags-from-image-label": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 4, + "mean": 1.5714285714285714, + "min": 0, + "samples": 7, + "stdev": 1.3972762620115438 + } + } + }, + "build/apply-tags/step-apply-additional-tags-from-parameter": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/build-image-index/step-build": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/build-image-index/step-create-sbom": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/build-image-index/step-upload-sbom": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/buildah/step-build": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 37, + "mean": 33.42857142857143, + "min": 30, + "samples": 7, + "stdev": 2.370453040886408 + } + } + }, + "build/buildah/step-icm": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/buildah/step-prepare-sboms": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 0.14285714285714285, + "min": 0, + "samples": 7, + "stdev": 0.37796447300922725 + } + } + }, + "build/buildah/step-push": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 17, + "mean": 11, + "min": 7, + "samples": 7, + "stdev": 4.509249752822894 + } + } + }, + "build/buildah/step-sbom-syft-generate": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 8, + "mean": 7.142857142857143, + "min": 7, + "samples": 7, + "stdev": 0.37796447300922725 + } + } + }, + "build/buildah/step-upload-sbom": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 5, + "mean": 3.5714285714285716, + "min": 3, + "samples": 7, + "stdev": 0.7867957924694432 + } + } + }, + "build/clair-scan/step-conftest-vulnerabilities": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/clair-scan/step-get-image-manifests": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 2, + "mean": 1.625, + "min": 1, + "samples": 8, + "stdev": 0.5175491695067657 + } + } + }, + "build/clair-scan/step-get-vulnerabilities": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 70, + "mean": 32.25, + "min": 24, + "samples": 8, + "stdev": 15.415669208401468 + } + } + }, + "build/clair-scan/step-oci-attach-report": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 7, + "mean": 2.875, + "min": 2, + "samples": 8, + "stdev": 1.7268882005337975 + } + } + }, + "build/clamav-scan/step-extract-and-scan-image": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 51, + "mean": 46.125, + "min": 44, + "samples": 8, + "stdev": 2.4748737341529163 + } + } + }, + "build/clamav-scan/step-upload": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 8, + "mean": 3.75, + "min": 2, + "samples": 8, + "stdev": 2.187627547301936 + } + } + }, + "build/coverity-availability-check/step-coverity-availability-check": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/deprecated-image-check/step-check-images": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 6, + "mean": 3.75, + "min": 2, + "samples": 8, + "stdev": 1.164964745021435 + } + } + }, + "build/ecosystem-cert-preflight-checks/step-check-container": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 16, + "mean": 12.125, + "min": 10, + "samples": 8, + "stdev": 1.9594095320493146 + } + } + }, + "build/ecosystem-cert-preflight-checks/step-gather-pflt-results": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 0.25, + "min": 0, + "samples": 8, + "stdev": 0.4629100498862757 + } + } + }, + "build/git-clone/step-clone": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 2, + "mean": 1.5, + "min": 1, + "samples": 8, + "stdev": 0.5345224838248488 + } + } + }, + "build/git-clone/step-symlink-check": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 0.125, + "min": 0, + "samples": 8, + "stdev": 0.3535533905932738 + } + } + }, + "build/init/step-init": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/push-dockerfile/step-push": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 6, + "mean": 2.7142857142857144, + "min": 1, + "samples": 7, + "stdev": 1.6035674514745464 + } + } + }, + "build/rpms-signature-scan/step-output-results": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/rpms-signature-scan/step-rpms-signature-scan": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 6, + "mean": 3.875, + "min": 2, + "samples": 8, + "stdev": 1.246423454758225 + } + } + }, + "build/sast-shell-check/step-sast-shell-check": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 1, + "min": 1, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/sast-shell-check/step-upload": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 5, + "mean": 2.2857142857142856, + "min": 1, + "samples": 7, + "stdev": 1.2535663410560174 + } + } + }, + "build/sast-snyk-check/step-sast-snyk-check": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 0.875, + "min": 0, + "samples": 8, + "stdev": 0.3535533905932738 + } + } + }, + "build/sast-snyk-check/step-upload": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/sast-unicode-check/step-sast-unicode-check": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 1, + "mean": 1, + "min": 1, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/sast-unicode-check/step-upload": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 6, + "mean": 3.875, + "min": 1, + "samples": 8, + "stdev": 1.8850918886280925 + } + } + }, + "build/show-sbom/step-show-sbom": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 8, + "mean": 2.75, + "min": 1, + "samples": 8, + "stdev": 2.49284690951645 + } + } + }, + "build/summary/step-appstudio-summary": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "test/test-output/step-unnamed-0": { + "failed": { + "duration": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 24, + "stdev": 0.0 + } + } + } + }, + "taskruns": { + "build/apply-tags": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 10, + "mean": 7, + "min": 5, + "samples": 7, + "stdev": 1.9148542155126762 + }, + "idle": { + "max": 7, + "mean": 5.428571428571429, + "min": 4, + "samples": 7, + "stdev": 1.2724180205607036 + }, + "running": { + "max": 10, + "mean": 7, + "min": 5, + "samples": 7, + "stdev": 1.9148542155126762 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/build-image-index": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 8, + "mean": 5.5, + "min": 4, + "samples": 8, + "stdev": 1.3093073414159544 + }, + "idle": { + "max": 8, + "mean": 5.5, + "min": 4, + "samples": 8, + "stdev": 1.3093073414159544 + }, + "running": { + "max": 8, + "mean": 5.375, + "min": 4, + "samples": 8, + "stdev": 1.3024701806293193 + }, + "scheduled": { + "max": 1, + "mean": 0.125, + "min": 0, + "samples": 8, + "stdev": 0.3535533905932738 + } + } + }, + "build/buildah": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 120, + "mean": 79.57142857142857, + "min": 64, + "samples": 7, + "stdev": 19.190771591723827 + }, + "idle": { + "max": 65, + "mean": 24.285714285714285, + "min": 12, + "samples": 7, + "stdev": 18.318608805567866 + }, + "running": { + "max": 120, + "mean": 79.57142857142857, + "min": 64, + "samples": 7, + "stdev": 19.190771591723827 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/clair-scan": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 110, + "mean": 48.625, + "min": 36, + "samples": 8, + "stdev": 24.933841031245638 + }, + "idle": { + "max": 36, + "mean": 11.875, + "min": 7, + "samples": 8, + "stdev": 9.789754119194502 + }, + "running": { + "max": 110, + "mean": 48.625, + "min": 36, + "samples": 8, + "stdev": 24.933841031245638 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/clamav-scan": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 74, + "mean": 59.5, + "min": 52, + "samples": 8, + "stdev": 7.521398046336104 + }, + "idle": { + "max": 19, + "mean": 9.625, + "min": 5, + "samples": 8, + "stdev": 4.838461975226662 + }, + "running": { + "max": 74, + "mean": 59.5, + "min": 52, + "samples": 8, + "stdev": 7.521398046336104 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/coverity-availability-check": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 7, + "mean": 6.142857142857143, + "min": 5, + "samples": 7, + "stdev": 0.6900655593423543 + }, + "idle": { + "max": 7, + "mean": 6.142857142857143, + "min": 5, + "samples": 7, + "stdev": 0.6900655593423543 + }, + "running": { + "max": 7, + "mean": 6.142857142857143, + "min": 5, + "samples": 7, + "stdev": 0.6900655593423543 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/deprecated-image-check": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 10, + "mean": 9, + "min": 8, + "samples": 8, + "stdev": 0.7559289460184545 + }, + "idle": { + "max": 7, + "mean": 5.25, + "min": 4, + "samples": 8, + "stdev": 1.0350983390135313 + }, + "running": { + "max": 10, + "mean": 9, + "min": 8, + "samples": 8, + "stdev": 0.7559289460184545 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/ecosystem-cert-preflight-checks": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 22, + "mean": 19.375, + "min": 16, + "samples": 8, + "stdev": 2.615202805574687 + }, + "idle": { + "max": 11, + "mean": 7, + "min": 5, + "samples": 8, + "stdev": 2.138089935299395 + }, + "running": { + "max": 22, + "mean": 19.375, + "min": 16, + "samples": 8, + "stdev": 2.615202805574687 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/git-clone": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 24, + "mean": 18.625, + "min": 15, + "samples": 8, + "stdev": 2.825268634509444 + }, + "idle": { + "max": 23, + "mean": 17, + "min": 14, + "samples": 8, + "stdev": 2.9760952365713798 + }, + "running": { + "max": 24, + "mean": 18.5, + "min": 15, + "samples": 8, + "stdev": 2.7774602993176543 + }, + "scheduled": { + "max": 1, + "mean": 0.125, + "min": 0, + "samples": 8, + "stdev": 0.3535533905932738 + } + } + }, + "build/init": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 6, + "mean": 5.375, + "min": 5, + "samples": 8, + "stdev": 0.5175491695067657 + }, + "idle": { + "max": 6, + "mean": 5.375, + "min": 5, + "samples": 8, + "stdev": 0.5175491695067657 + }, + "running": { + "max": 6, + "mean": 5.25, + "min": 5, + "samples": 8, + "stdev": 0.4629100498862757 + }, + "scheduled": { + "max": 1, + "mean": 0.125, + "min": 0, + "samples": 8, + "stdev": 0.3535533905932738 + } + } + }, + "build/push-dockerfile": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 19, + "mean": 16.428571428571427, + "min": 14, + "samples": 7, + "stdev": 1.618347187425374 + }, + "idle": { + "max": 16, + "mean": 13.714285714285714, + "min": 10, + "samples": 7, + "stdev": 2.0586634591635513 + }, + "running": { + "max": 19, + "mean": 16.428571428571427, + "min": 14, + "samples": 7, + "stdev": 1.618347187425374 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/rpms-signature-scan": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 32, + "mean": 12.875, + "min": 9, + "samples": 8, + "stdev": 7.754031209635412 + }, + "idle": { + "max": 27, + "mean": 9, + "min": 5, + "samples": 8, + "stdev": 7.329003050503235 + }, + "running": { + "max": 32, + "mean": 12.875, + "min": 9, + "samples": 8, + "stdev": 7.754031209635412 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/sast-shell-check": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 20, + "mean": 17.142857142857142, + "min": 14, + "samples": 7, + "stdev": 2.5448360411214073 + }, + "idle": { + "max": 17, + "mean": 13.857142857142858, + "min": 8, + "samples": 7, + "stdev": 3.4364987719368982 + }, + "running": { + "max": 20, + "mean": 17.142857142857142, + "min": 14, + "samples": 7, + "stdev": 2.5448360411214073 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 7, + "stdev": 0.0 + } + } + }, + "build/sast-snyk-check": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 19, + "mean": 14.625, + "min": 9, + "samples": 8, + "stdev": 3.1594529363709247 + }, + "idle": { + "max": 19, + "mean": 13.75, + "min": 8, + "samples": 8, + "stdev": 3.370036032024414 + }, + "running": { + "max": 19, + "mean": 14.625, + "min": 9, + "samples": 8, + "stdev": 3.1594529363709247 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/sast-unicode-check": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 24, + "mean": 18.75, + "min": 13, + "samples": 8, + "stdev": 4.200340122282616 + }, + "idle": { + "max": 17, + "mean": 13.875, + "min": 7, + "samples": 8, + "stdev": 3.563204817496262 + }, + "running": { + "max": 24, + "mean": 18.75, + "min": 13, + "samples": 8, + "stdev": 4.200340122282616 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/show-sbom": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 13, + "mean": 7.75, + "min": 5, + "samples": 8, + "stdev": 2.6049403612586386 + }, + "idle": { + "max": 7, + "mean": 5, + "min": 4, + "samples": 8, + "stdev": 0.9258200997725514 + }, + "running": { + "max": 13, + "mean": 7.75, + "min": 5, + "samples": 8, + "stdev": 2.6049403612586386 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "build/summary": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 14, + "mean": 11, + "min": 9, + "samples": 8, + "stdev": 2.32992949004287 + }, + "idle": { + "max": 14, + "mean": 11, + "min": 9, + "samples": 8, + "stdev": 2.32992949004287 + }, + "running": { + "max": 14, + "mean": 11, + "min": 9, + "samples": 8, + "stdev": 2.32992949004287 + }, + "scheduled": { + "max": 0, + "mean": 0, + "min": 0, + "samples": 8, + "stdev": 0.0 + } + } + }, + "test/test-output": { + "failed": { + "duration": { + "samples": 0 + }, + "idle": { + "samples": 0 + }, + "running": { + "samples": 0 + }, + "scheduled": { + "samples": 0 + } + }, + "passed": { + "duration": { + "max": 24, + "mean": 8.666666666666666, + "min": 4, + "samples": 24, + "stdev": 6.091095901015048 + }, + "idle": { + "max": 24, + "mean": 8.666666666666666, + "min": 4, + "samples": 24, + "stdev": 6.091095901015048 + }, + "running": { + "max": 24, + "mean": 8.625, + "min": 4, + "samples": 24, + "stdev": 6.0778464435443205 + }, + "scheduled": { + "max": 1, + "mean": 0.041666666666666664, + "min": 0, + "samples": 24, + "stdev": 0.2041241452319315 + } + } + } + } + } + }, + "measurements": { + "HandleUser": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.106332, + "mean": 0.10297240909090909, + "min": 0.102207, + "samples": 22 + }, + "when": { + "max": "2025-02-27T07:29:38.916159+00:00", + "mean": "2025-02-26T06:21:14.789978+00:00", + "min": "2025-02-25T14:33:32.203209+00:00", + "span": 147366.71295 + } + } + }, + "KPI": { + "errors": 2, + "mean": 265.78953718931064 + }, + "createApplication": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.140243, + "mean": 0.09747566666666667, + "min": 0.06337, + "samples": 21 + }, + "when": { + "max": "2025-02-27T07:29:39.098700+00:00", + "mean": "2025-02-26T06:04:02.446122+00:00", + "min": "2025-02-25T14:33:32.412278+00:00", + "span": 147366.686422 + } + } + }, + "createComponent": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 4.100836, + "mean": 3.543904761904762, + "min": 3.087928, + "samples": 21 + }, + "when": { + "max": "2025-02-27T07:30:02.378217+00:00", + "mean": "2025-02-26T06:04:23.309789+00:00", + "min": "2025-02-25T14:33:56.706255+00:00", + "span": 147365.671962 + } + } + }, + "createIntegrationTestScenario": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.072855, + "mean": 0.04223152380952381, + "min": 0.034384, + "samples": 21 + }, + "when": { + "max": "2025-02-27T07:29:39.150324+00:00", + "mean": "2025-02-26T06:04:02.502418+00:00", + "min": "2025-02-25T14:33:32.496315+00:00", + "span": 147366.654009 + } + } + }, + "validateApplication": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.023302, + "mean": 0.013954666666666667, + "min": 0.011048, + "samples": 21 + }, + "when": { + "max": "2025-02-27T07:29:39.111940+00:00", + "mean": "2025-02-26T06:04:02.460108+00:00", + "min": "2025-02-25T14:33:32.423375+00:00", + "span": 147366.688565 + } + } + }, + "validateIntegrationTestScenario": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 20.024498, + "mean": 17.160412904761905, + "min": 0.014145, + "samples": 21 + }, + "when": { + "max": "2025-02-27T07:29:59.168111+00:00", + "mean": "2025-02-26T06:04:19.662861+00:00", + "min": "2025-02-25T14:33:52.512014+00:00", + "span": 147366.656097 + } + } + }, + "validatePipelineRunCondition": { + "error_rate": 0.13333333333333333, + "fail": { + "duration": { + "max": 160.046586, + "mean": 130.045698, + "min": 100.04481, + "samples": 2 + }, + "when": { + "max": "2025-02-25T15:28:07.010725+00:00", + "mean": "2025-02-25T15:07:43.765700+00:00", + "min": "2025-02-25T14:47:20.520675+00:00", + "span": 2446.49005 + } + }, + "pass": { + "duration": { + "max": 380.047657, + "mean": 230.82191230769232, + "min": 160.054488, + "samples": 13 + }, + "when": { + "max": "2025-02-26T14:11:54.148043+00:00", + "mean": "2025-02-26T04:08:25.012730+00:00", + "min": "2025-02-25T14:39:16.811313+00:00", + "span": 84757.33673 + } + } + }, + "validatePipelineRunCreation": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.077911, + "mean": 0.056787333333333335, + "min": 0.045617, + "samples": 15 + }, + "when": { + "max": "2025-02-26T14:08:54.091286+00:00", + "mean": "2025-02-26T02:20:42.128011+00:00", + "min": "2025-02-25T14:34:16.767706+00:00", + "span": 84877.32358 + } + } + }, + "validatePipelineRunSignature": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 20.057861, + "mean": 7.734745384615385, + "min": 0.029584, + "samples": 13 + }, + "when": { + "max": "2025-02-26T14:12:14.189636+00:00", + "mean": "2025-02-26T04:08:32.747508+00:00", + "min": "2025-02-25T14:39:36.864049+00:00", + "span": 84757.325587 + } + } + }, + "validateSnapshotCreation": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.028322, + "mean": 0.015941153846153848, + "min": 0.012823, + "samples": 13 + }, + "when": { + "max": "2025-02-26T14:12:14.204033+00:00", + "mean": "2025-02-26T04:08:32.763500+00:00", + "min": "2025-02-25T14:39:36.876930+00:00", + "span": 84757.327103 + } + } + }, + "validateTestPipelineRunCondition": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 20.056878, + "mean": 6.179364769230769, + "min": 0.016686, + "samples": 13 + }, + "when": { + "max": "2025-02-26T14:12:14.239418+00:00", + "mean": "2025-02-26T04:08:38.962750+00:00", + "min": "2025-02-25T14:39:36.910503+00:00", + "span": 84757.328915 + } + } + }, + "validateTestPipelineRunCreation": { + "error_rate": 0.0, + "fail": { + "duration": { + "samples": 0 + }, + "when": {} + }, + "pass": { + "duration": { + "max": 0.023854, + "mean": 0.019834307692307693, + "min": 0.016821, + "samples": 13 + }, + "when": { + "max": "2025-02-26T14:12:14.221953+00:00", + "mean": "2025-02-26T04:08:32.783361+00:00", + "min": "2025-02-25T14:39:36.893785+00:00", + "span": 84757.328168 + } + } + } + } + }, + "started": "2025-02-27T07:29:25,261140292+00:00" +} \ No newline at end of file diff --git a/DELME/measurements_cluster_cpu_usage_seconds_total_rate.csv b/DELME/measurements_cluster_cpu_usage_seconds_total_rate.csv new file mode 100644 index 00000000..9f1a48a5 --- /dev/null +++ b/DELME/measurements_cluster_cpu_usage_seconds_total_rate.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_cpu_usage_seconds_total_rate +1698562800,17.105312950202077 +1698562815,15.157145630211222 +1698562830,15.157145630211222 +1698562845,19.4157626476966 +1698562860,19.4157626476966 +1698562875,17.107327462801695 +1698562890,17.107327462801695 +1698562905,13.759914315487771 +1698562920,13.759914315487771 +1698562935,15.803033571788406 +1698562950,15.803033571788406 +1698562965,15.981675511394727 +1698562980,15.981675511394727 +1698562995,15.915285517729501 +1698563010,15.915285517729501 +1698563025,14.783625195477825 +1698563040,14.783625195477825 +1698563055,16.026150000362424 +1698563070,16.026150000362424 +1698563085,16.314978855105508 +1698563100,16.314978855105508 +1698563115,14.195491272625802 +1698563130,14.195491272625802 +1698563145,14.439392751830102 +1698563160,14.439392751830102 +1698563175,14.393816443101864 +1698563190,14.393816443101864 +1698563205,11.974644726333159 +1698563220,11.974644726333159 +1698563235,15.195653694970094 +1698563250,15.195653694970094 +1698563265,12.565355629565051 +1698563280,12.565355629565051 +1698563295,14.420666147720766 +1698563310,14.420666147720766 +1698563325,14.415172064673337 +1698563340,14.415172064673337 +1698563355,12.550134344708585 +1698563370,12.550134344708585 +1698563385,15.578046407902907 +1698563400,15.578046407902907 +1698563415,12.702104107036442 +1698563430,12.702104107036442 +1698563445,14.167250137127592 +1698563460,14.167250137127592 +1698563475,13.494265791066217 +1698563490,13.494265791066217 +1698563505,12.856772802437758 +1698563520,12.856772802437758 +1698563535,12.82224680446754 +1698563550,12.82224680446754 +1698563565,14.55036973262372 +1698563580,14.55036973262372 +1698563595,13.504658523298993 +1698563610,13.504658523298993 +1698563625,13.745143397758193 +1698563640,13.745143397758193 +1698563655,13.443220717199129 +1698563670,13.443220717199129 +1698563685,13.949547464768887 +1698563700,13.949547464768887 diff --git a/DELME/measurements_cluster_disk_throughput_total.csv b/DELME/measurements_cluster_disk_throughput_total.csv new file mode 100644 index 00000000..0bb7cff4 --- /dev/null +++ b/DELME/measurements_cluster_disk_throughput_total.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_disk_throughput_total +1698562800,19828742.452728476 +1698562815,19956244.859259248 +1698562830,32755697.38540454 +1698562845,32627217.622441575 +1698562860,33879405.98518517 +1698562875,34051860.85925925 +1698562890,34481555.86979336 +1698562905,34515046.35868225 +1698562920,33768636.443120524 +1698562935,33616039.58386126 +1698562950,34390131.2059138 +1698562965,34563289.605913796 +1698562980,34724754.01481481 +1698562995,34590331.25925925 +1698563010,34762107.259259254 +1698563025,34719637.8074074 +1698563040,34680684.08888888 +1698563055,34863344.82962963 +1698563070,35232557.511111096 +1698563085,35202099.19999999 +1698563100,22271863.206869353 +1698563115,22315091.177239724 +1698563130,21104054.04444443 +1698563145,20909029.451851837 +1698563160,20642592.226228014 +1698563175,20812178.004005797 +1698563190,21033572.5037037 +1698563205,20976296.770370364 +1698563220,20328825.362962954 +1698563235,20561679.170370363 +1698563250,21335489.422222223 +1698563265,21274288.355555553 +1698563280,21011654.20027592 +1698563295,21165495.029905554 +1698563310,21093585.148869287 +1698563325,20971592.615535952 +1698563340,20575103.908957385 +1698563355,20362688.383031465 +1698563370,20374180.977777775 +1698563385,20381984.237037033 +1698563400,20286549.333333332 +1698563415,20355445.570370365 +1698563430,20346570.90370369 +1698563445,20128604.918518513 +1698563460,19726402.37037037 +1698563475,19726271.52592592 +1698563490,19795696.82962962 +1698563505,19326579.674074065 +1698563520,18305477.21481481 +1698563535,18543485.155555546 +1698563550,19433830.942776226 +1698563565,19455879.17981327 +1698563580,19408704.861329902 +1698563595,19508948.77244101 +1698563610,19531956.241510656 +1698563625,19595281.16002917 +1698563640,19640896.474074062 +1698563655,19820104.05925925 +1698563670,19919879.585185174 +1698563685,19802440.059259247 +1698563700,19552931.08148147 diff --git a/DELME/measurements_cluster_memory_usage_rss_total.csv b/DELME/measurements_cluster_memory_usage_rss_total.csv new file mode 100644 index 00000000..a9bb3fbe --- /dev/null +++ b/DELME/measurements_cluster_memory_usage_rss_total.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_memory_usage_rss_total +1698562800,93735653376.0 +1698562815,93773635584.0 +1698562830,93607755776.0 +1698562845,93566263296.0 +1698562860,93099159552.0 +1698562875,92629274624.0 +1698562890,92710813696.0 +1698562905,92585607168.0 +1698562920,93185138688.0 +1698562935,92880412672.0 +1698562950,92922884096.0 +1698562965,92631646208.0 +1698562980,92635709440.0 +1698562995,92674224128.0 +1698563010,92210102272.0 +1698563025,92274540544.0 +1698563040,91965612032.0 +1698563055,91988271104.0 +1698563070,91622912000.0 +1698563085,91877220352.0 +1698563100,92095918080.0 +1698563115,91753140224.0 +1698563130,90921975808.0 +1698563145,91356262400.0 +1698563160,91927314432.0 +1698563175,91983634432.0 +1698563190,91747966976.0 +1698563205,91674550272.0 +1698563220,91853729792.0 +1698563235,91878006784.0 +1698563250,92040290304.0 +1698563265,92038610944.0 +1698563280,91728556032.0 +1698563295,91739033600.0 +1698563310,92040544256.0 +1698563325,92094767104.0 +1698563340,92031680512.0 +1698563355,92079362048.0 +1698563370,91736588288.0 +1698563385,91720761344.0 +1698563400,91946520576.0 +1698563415,91945545728.0 +1698563430,92355506176.0 +1698563445,92389793792.0 +1698563460,92000387072.0 +1698563475,91975323648.0 +1698563490,92212809728.0 +1698563505,92272095232.0 +1698563520,92593332224.0 +1698563535,92498538496.0 +1698563550,92309749760.0 +1698563565,92337692672.0 +1698563580,92223025152.0 +1698563595,92218085376.0 +1698563610,92540166144.0 +1698563625,92562518016.0 +1698563640,92593704960.0 +1698563655,92615258112.0 +1698563670,92787073024.0 +1698563685,92899577856.0 +1698563700,92749586432.0 diff --git a/DELME/measurements_cluster_nodes_worker_count.csv b/DELME/measurements_cluster_nodes_worker_count.csv new file mode 100644 index 00000000..966acbf0 --- /dev/null +++ b/DELME/measurements_cluster_nodes_worker_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_nodes_worker_count +1698562800,9.0 +1698562815,9.0 +1698562830,9.0 +1698562845,9.0 +1698562860,9.0 +1698562875,9.0 +1698562890,9.0 +1698562905,9.0 +1698562920,9.0 +1698562935,9.0 +1698562950,9.0 +1698562965,9.0 +1698562980,9.0 +1698562995,9.0 +1698563010,9.0 +1698563025,9.0 +1698563040,9.0 +1698563055,9.0 +1698563070,9.0 +1698563085,9.0 +1698563100,9.0 +1698563115,9.0 +1698563130,9.0 +1698563145,9.0 +1698563160,9.0 +1698563175,9.0 +1698563190,9.0 +1698563205,9.0 +1698563220,9.0 +1698563235,9.0 +1698563250,9.0 +1698563265,9.0 +1698563280,9.0 +1698563295,9.0 +1698563310,9.0 +1698563325,9.0 +1698563340,9.0 +1698563355,9.0 +1698563370,9.0 +1698563385,9.0 +1698563400,9.0 +1698563415,9.0 +1698563430,9.0 +1698563445,9.0 +1698563460,9.0 +1698563475,9.0 +1698563490,9.0 +1698563505,9.0 +1698563520,9.0 +1698563535,9.0 +1698563550,9.0 +1698563565,9.0 +1698563580,9.0 +1698563595,9.0 +1698563610,9.0 +1698563625,9.0 +1698563640,9.0 +1698563655,9.0 +1698563670,9.0 +1698563685,9.0 +1698563700,9.0 diff --git a/DELME/measurements_cluster_pods_count.csv b/DELME/measurements_cluster_pods_count.csv new file mode 100644 index 00000000..e898ad9f --- /dev/null +++ b/DELME/measurements_cluster_pods_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_pods_count +1698562800,712.0 +1698562815,712.0 +1698562830,715.0 +1698562845,715.0 +1698562860,715.0 +1698562875,715.0 +1698562890,713.0 +1698562905,713.0 +1698562920,713.0 +1698562935,713.0 +1698562950,714.0 +1698562965,714.0 +1698562980,714.0 +1698562995,714.0 +1698563010,712.0 +1698563025,712.0 +1698563040,712.0 +1698563055,712.0 +1698563070,713.0 +1698563085,713.0 +1698563100,713.0 +1698563115,713.0 +1698563130,712.0 +1698563145,712.0 +1698563160,712.0 +1698563175,712.0 +1698563190,713.0 +1698563205,713.0 +1698563220,713.0 +1698563235,713.0 +1698563250,712.0 +1698563265,712.0 +1698563280,712.0 +1698563295,712.0 +1698563310,713.0 +1698563325,713.0 +1698563340,713.0 +1698563355,713.0 +1698563370,712.0 +1698563385,712.0 +1698563400,712.0 +1698563415,712.0 +1698563430,713.0 +1698563445,713.0 +1698563460,713.0 +1698563475,713.0 +1698563490,712.0 +1698563505,712.0 +1698563520,712.0 +1698563535,712.0 +1698563550,713.0 +1698563565,713.0 +1698563580,713.0 +1698563595,713.0 +1698563610,712.0 +1698563625,712.0 +1698563640,712.0 +1698563655,712.0 +1698563670,714.0 +1698563685,714.0 +1698563700,714.0 diff --git a/DELME/measurements_cluster_running_pods_on_workers_count.csv b/DELME/measurements_cluster_running_pods_on_workers_count.csv new file mode 100644 index 00000000..1ea851a0 --- /dev/null +++ b/DELME/measurements_cluster_running_pods_on_workers_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.cluster_running_pods_on_workers_count +1698562800,401.0 +1698562815,401.0 +1698562830,402.0 +1698562845,401.0 +1698562860,401.0 +1698562875,401.0 +1698562890,398.0 +1698562905,398.0 +1698562920,398.0 +1698562935,398.0 +1698562950,399.0 +1698562965,398.0 +1698562980,398.0 +1698562995,398.0 +1698563010,398.0 +1698563025,398.0 +1698563040,398.0 +1698563055,398.0 +1698563070,399.0 +1698563085,398.0 +1698563100,398.0 +1698563115,398.0 +1698563130,398.0 +1698563145,398.0 +1698563160,398.0 +1698563175,398.0 +1698563190,399.0 +1698563205,398.0 +1698563220,398.0 +1698563235,398.0 +1698563250,398.0 +1698563265,398.0 +1698563280,398.0 +1698563295,398.0 +1698563310,399.0 +1698563325,398.0 +1698563340,398.0 +1698563355,398.0 +1698563370,398.0 +1698563385,398.0 +1698563400,398.0 +1698563415,398.0 +1698563430,399.0 +1698563445,398.0 +1698563460,398.0 +1698563475,398.0 +1698563490,398.0 +1698563505,398.0 +1698563520,398.0 +1698563535,398.0 +1698563550,399.0 +1698563565,398.0 +1698563580,398.0 +1698563595,398.0 +1698563610,398.0 +1698563625,398.0 +1698563640,398.0 +1698563655,398.0 +1698563670,399.0 +1698563685,398.0 +1698563700,398.0 diff --git a/DELME/measurements_etcd_etcd_request_duration_seconds_average.csv b/DELME/measurements_etcd_etcd_request_duration_seconds_average.csv new file mode 100644 index 00000000..c8b3ce09 --- /dev/null +++ b/DELME/measurements_etcd_etcd_request_duration_seconds_average.csv @@ -0,0 +1,62 @@ +timestamp,measurements.etcd_etcd_request_duration_seconds_average +1698562800,0.002312911373130061 +1698562815,0.002316922182703827 +1698562830,0.0023216314674395088 +1698562845,0.0023293304913434417 +1698562860,0.002330461204635945 +1698562875,0.0023484889350565774 +1698562890,0.0023553693529017213 +1698562905,0.0023500475834855716 +1698562920,0.002351779544009023 +1698562935,0.0023654128185171606 +1698562950,0.002378004325119549 +1698562965,0.002374841143520472 +1698562980,0.0023704180732064288 +1698562995,0.0023681743185857236 +1698563010,0.0023693777859074075 +1698563025,0.0023461963967787323 +1698563040,0.0023374229792219596 +1698563055,0.002365378470191423 +1698563070,0.0023763230093297246 +1698563085,0.0023643811611102207 +1698563100,0.0023584029105001006 +1698563115,0.002351167242765439 +1698563130,0.002353508975938061 +1698563145,0.002338897594530227 +1698563160,0.002340795512972262 +1698563175,0.0023540016816585405 +1698563190,0.002357693755423384 +1698563205,0.0023454041081434358 +1698563220,0.002333870695120055 +1698563235,0.002344385830727448 +1698563250,0.0023587752853893793 +1698563265,0.0023420619493184918 +1698563280,0.0023342945259239526 +1698563295,0.0023524862101314626 +1698563310,0.002365460141211915 +1698563325,0.002348221931905284 +1698563340,0.002332621391262931 +1698563355,0.0023313884194421115 +1698563370,0.0023330445163706067 +1698563385,0.0023356125971757532 +1698563400,0.0023353973905367222 +1698563415,0.0023566162630029785 +1698563430,0.0023565574203249272 +1698563445,0.002340424479476883 +1698563460,0.0023331713302504574 +1698563475,0.0023347287723041693 +1698563490,0.002336052189827706 +1698563505,0.002327545254131022 +1698563520,0.0023168839869638544 +1698563535,0.002331320471366188 +1698563550,0.0023421573336280185 +1698563565,0.0023314688234763203 +1698563580,0.0023175750581246454 +1698563595,0.0023070063947108798 +1698563610,0.002299894574330571 +1698563625,0.002296418560183471 +1698563640,0.002293611619803514 +1698563655,0.0022982613981083035 +1698563670,0.002299202134343484 +1698563685,0.0022745823904629214 +1698563700,0.0022699344025071387 diff --git a/DELME/measurements_scheduler_pending_pods_count.csv b/DELME/measurements_scheduler_pending_pods_count.csv new file mode 100644 index 00000000..b765bc70 --- /dev/null +++ b/DELME/measurements_scheduler_pending_pods_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.scheduler_pending_pods_count +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_storage_count_attachable_volumes_in_use.csv b/DELME/measurements_storage_count_attachable_volumes_in_use.csv new file mode 100644 index 00000000..d1a7be2a --- /dev/null +++ b/DELME/measurements_storage_count_attachable_volumes_in_use.csv @@ -0,0 +1,62 @@ +timestamp,measurements.storage_count_attachable_volumes_in_use +1698562800,7.0 +1698562815,7.0 +1698562830,7.0 +1698562845,7.0 +1698562860,7.0 +1698562875,7.0 +1698562890,7.0 +1698562905,7.0 +1698562920,7.0 +1698562935,7.0 +1698562950,7.0 +1698562965,7.0 +1698562980,7.0 +1698562995,7.0 +1698563010,7.0 +1698563025,7.0 +1698563040,7.0 +1698563055,7.0 +1698563070,7.0 +1698563085,7.0 +1698563100,7.0 +1698563115,7.0 +1698563130,7.0 +1698563145,7.0 +1698563160,7.0 +1698563175,7.0 +1698563190,7.0 +1698563205,7.0 +1698563220,7.0 +1698563235,7.0 +1698563250,7.0 +1698563265,7.0 +1698563280,7.0 +1698563295,7.0 +1698563310,7.0 +1698563325,7.0 +1698563340,7.0 +1698563355,7.0 +1698563370,7.0 +1698563385,7.0 +1698563400,7.0 +1698563415,7.0 +1698563430,7.0 +1698563445,7.0 +1698563460,7.0 +1698563475,7.0 +1698563490,7.0 +1698563505,7.0 +1698563520,7.0 +1698563535,7.0 +1698563550,7.0 +1698563565,7.0 +1698563580,7.0 +1698563595,7.0 +1698563610,7.0 +1698563625,7.0 +1698563640,7.0 +1698563655,7.0 +1698563670,7.0 +1698563685,7.0 +1698563700,7.0 diff --git a/DELME/measurements_tekton_pipelines_controller_running_pipelineruns_count.csv b/DELME/measurements_tekton_pipelines_controller_running_pipelineruns_count.csv new file mode 100644 index 00000000..9a21cf4b --- /dev/null +++ b/DELME/measurements_tekton_pipelines_controller_running_pipelineruns_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.tekton_pipelines_controller_running_pipelineruns_count +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_node_count.csv b/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_node_count.csv new file mode 100644 index 00000000..ad21084e --- /dev/null +++ b/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_node_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.tekton_tekton_pipelines_controller_running_taskruns_throttled_by_node_count +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_quota_count.csv b/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_quota_count.csv new file mode 100644 index 00000000..6af4fada --- /dev/null +++ b/DELME/measurements_tekton_tekton_pipelines_controller_running_taskruns_throttled_by_quota_count.csv @@ -0,0 +1,62 @@ +timestamp,measurements.tekton_tekton_pipelines_controller_running_taskruns_throttled_by_quota_count +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_tekton_tekton_pipelines_controller_workqueue_depth.csv b/DELME/measurements_tekton_tekton_pipelines_controller_workqueue_depth.csv new file mode 100644 index 00000000..8a75e64a --- /dev/null +++ b/DELME/measurements_tekton_tekton_pipelines_controller_workqueue_depth.csv @@ -0,0 +1,62 @@ +timestamp,measurements.tekton_tekton_pipelines_controller_workqueue_depth +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_token_pool_rate_primary.csv b/DELME/measurements_token_pool_rate_primary.csv new file mode 100644 index 00000000..c1a649d5 --- /dev/null +++ b/DELME/measurements_token_pool_rate_primary.csv @@ -0,0 +1,62 @@ +timestamp,measurements.token_pool_rate_primary +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/measurements_token_pool_rate_secondary.csv b/DELME/measurements_token_pool_rate_secondary.csv new file mode 100644 index 00000000..803c23dc --- /dev/null +++ b/DELME/measurements_token_pool_rate_secondary.csv @@ -0,0 +1,62 @@ +timestamp,measurements.token_pool_rate_secondary +1698562800,0.0 +1698562815,0.0 +1698562830,0.0 +1698562845,0.0 +1698562860,0.0 +1698562875,0.0 +1698562890,0.0 +1698562905,0.0 +1698562920,0.0 +1698562935,0.0 +1698562950,0.0 +1698562965,0.0 +1698562980,0.0 +1698562995,0.0 +1698563010,0.0 +1698563025,0.0 +1698563040,0.0 +1698563055,0.0 +1698563070,0.0 +1698563085,0.0 +1698563100,0.0 +1698563115,0.0 +1698563130,0.0 +1698563145,0.0 +1698563160,0.0 +1698563175,0.0 +1698563190,0.0 +1698563205,0.0 +1698563220,0.0 +1698563235,0.0 +1698563250,0.0 +1698563265,0.0 +1698563280,0.0 +1698563295,0.0 +1698563310,0.0 +1698563325,0.0 +1698563340,0.0 +1698563355,0.0 +1698563370,0.0 +1698563385,0.0 +1698563400,0.0 +1698563415,0.0 +1698563430,0.0 +1698563445,0.0 +1698563460,0.0 +1698563475,0.0 +1698563490,0.0 +1698563505,0.0 +1698563520,0.0 +1698563535,0.0 +1698563550,0.0 +1698563565,0.0 +1698563580,0.0 +1698563595,0.0 +1698563610,0.0 +1698563625,0.0 +1698563640,0.0 +1698563655,0.0 +1698563670,0.0 +1698563685,0.0 +1698563700,0.0 diff --git a/DELME/my-tekton-perfscale-experiment-investigator_config.yaml b/DELME/my-tekton-perfscale-experiment-investigator_config.yaml new file mode 100644 index 00000000..4f774c9d --- /dev/null +++ b/DELME/my-tekton-perfscale-experiment-investigator_config.yaml @@ -0,0 +1,37 @@ +# Get historical test results from here +history: + type: sd_dir + dir: ../my-tekton-perfscale-experiment-results/ + matchers: | + name: "{{ current.get('name') }}" + result: PASS + parameters.locust.host: "{{ current.get('parameters.locust.host') }}" + #parameters.locust.num_clients: {{ current.get('parameters.locust.num_clients') }} + #parameters.locust.hatch_rate: {{ current.get('parameters.locust.hatch_rate') }} + #parameters.test.duration: {{ current.get('parameters.test.duration') }} + +# From here get current result that is supposed to be compared to historical results +current: + type: status_data + file: status-data.json + +# JSON paths we want to investigate +sets: + ###- measurements.rbac_all.cpu.mean + ###- measurements.rbac_all.memory.mean + ###- measurements.rbac-service.cpu.mean + ###- measurements.rbac-service.memory.mean + ###- measurements.rbac-scheduler-service.cpu.mean + ###- measurements.rbac-scheduler-service.memory.mean + ###- measurements.rbac-worker-service.cpu.mean + ###- measurements.rbac-worker-service.memory.mean + - results.requests.SUMMARY.total_rps + - results.requests.SUMMARY.median_response_time + - results.requests.SUMMARY.fail_ratio + - results.requests.SUMMARY.num_failures + - results.requests.SUMMARY.num_requests + +# Shuld we log our decisions somewhere? +decisions: + type: csv + filename: ../my-tekton-perfscale-experiment-results/decisions.csv diff --git a/DELME/opl/cluster_read_caps.yaml b/DELME/opl/cluster_read_caps.yaml new file mode 100644 index 00000000..e438c771 --- /dev/null +++ b/DELME/opl/cluster_read_caps.yaml @@ -0,0 +1,7 @@ +- name: measurements.capsules_data.load.load.shortterm[] + grafana_target: $Cloud.$Node.load.load.shortterm + grafana_enritchment: + capsule: "{{CAPSULE}}" +- name: measurements.capsules_data.memory.memory-used[] + grafana_target: $Cloud.$Node.memory.memory-used + grafana_include_vars: true diff --git a/DELME/opl/generators/inventory_egress_template.json.j2-3 b/DELME/opl/generators/inventory_egress_template.json.j2-3 new file mode 100644 index 00000000..ba29f521 --- /dev/null +++ b/DELME/opl/generators/inventory_egress_template.json.j2-3 @@ -0,0 +1,620 @@ +{ + "host": { + "account": "{{ account }}", + "ansible_host": null, + "bios_uuid": "{{ bios_uuid }}", + "created": "{{ nowz }}", + "culled_timestamp": "{{ tommorowz }}", + "display_name": "{{ fqdn }}", + "external_id": null, + "fqdn": "{{ fqdn }}", + "id": "{{ inventory_id }}", + "insights_id": "{{ insights_id }}", + "ip_addresses": [ + "{{ ipv4_addr }}", + "{{ ipv6_addr }}" + ], + "mac_addresses": [ + "{{ mac_addr }}", + "00:00:00:00:00:00" + ], + "per_reporter_staleness": { + "puptoo": { + "check_in_succeeded": true, + "last_check_in": "{{ nowz }}", + "stale_timestamp": "{{ tommorowz }}" + } + }, + "reporter": "puptoo", + "rhel_machine_id": null, + "satellite_id": null, + "stale_timestamp": "{{ tommorowz }}", + "stale_warning_timestamp": "{{ tommorowz }}", + "subscription_manager_id": "{{ subscription_manager_id }}", + "system_profile": { + "arch": "x86_64", + "bios_release_date": "04/01/2014", + "bios_vendor": "SeaBIOS", + "bios_version": "1.11.0-2.el7", + "captured_date": "2019-11-19T22:13:22+00:00", + "cores_per_socket": 1, + "cpu_flags": [ + "fpu", + "vme", + "de", + "pse", + "tsc", + "msr", + "pae", + "mce", + "cx8", + "apic", + "sep", + "mtrr", + "pge", + "mca", + "cmov", + "pat", + "pse36", + "clflush", + "mmx", + "fxsr", + "sse", + "sse2", + "ss", + "syscall", + "nx", + "pdpe1gb", + "rdtscp", + "lm", + "constant_tsc", + "rep_good", + "nopl", + "xtopology", + "eagerfpu", + "pni", + "pclmulqdq", + "vmx", + "ssse3", + "fma", + "cx16", + "pcid", + "sse4_1", + "sse4_2", + "x2apic", + "movbe", + "popcnt", + "tsc_deadline_timer", + "aes", + "xsave", + "avx", + "f16c", + "rdrand", + "hypervisor", + "lahf_lm", + "abm", + "3dnowprefetch", + "invpcid_single", + "ssbd", + "ibrs", + "ibpb", + "stibp", + "tpr_shadow", + "vnmi", + "flexpriority", + "ept", + "vpid", + "fsgsbase", + "tsc_adjust", + "bmi1", + "hle", + "avx2", + "smep", + "bmi2", + "erms", + "invpcid", + "rtm", + "mpx", + "avx512f", + "avx512dq", + "rdseed", + "adx", + "smap", + "clflushopt", + "clwb", + "avx512cd", + "avx512bw", + "avx512vl", + "xsaveopt", + "xsavec", + "xgetbv1", + "arat", + "pku", + "ospke", + "md_clear", + "spec_ctrl", + "intel_stibp" + ], + "enabled_services": [ + "auditd", + "auth-rpcgss-module", + "autovt@", + "brandbot", + "chrony-dnssrv@", + "chronyd", + "cloud-config", + "cloud-final", + "cloud-init-local", + "cloud-init", + "container-getty@", + "crond", + "dbus-org.freedesktop.hostname1", + "dbus-org.freedesktop.import1", + "dbus-org.freedesktop.locale1", + "dbus-org.freedesktop.login1", + "dbus-org.freedesktop.machine1", + "dbus-org.freedesktop.nm-dispatcher", + "dbus-org.freedesktop.timedate1", + "dbus", + "dracut-cmdline", + "dracut-initqueue", + "dracut-mount", + "dracut-pre-mount", + "dracut-pre-pivot", + "dracut-pre-trigger", + "dracut-pre-udev", + "dracut-shutdown", + "emergency", + "fstrim", + "getty@", + "halt-local", + "initrd-cleanup", + "initrd-parse-etc", + "initrd-switch-root", + "initrd-udevadm-cleanup-db", + "insights-client", + "irqbalance", + "kdump", + "kmod-static-nodes", + "messagebus", + "microcode", + "NetworkManager-dispatcher", + "NetworkManager-wait-online", + "NetworkManager", + "nfs-config", + "nfs-idmap", + "nfs-idmapd", + "nfs-lock", + "nfs-mountd", + "nfs-secure", + "nfs-utils", + "nfslock", + "ovirt-guest-agent", + "polkit", + "postfix", + "qemu-guest-agent", + "quotaon", + "rc-local", + "rescue", + "rhel-autorelabel-mark", + "rhel-autorelabel", + "rhel-configure", + "rhel-dmesg", + "rhel-domainname", + "rhel-import-state", + "rhel-loadmodules", + "rhel-readonly", + "rpc-gssd", + "rpc-statd-notify", + "rpc-statd", + "rpcbind", + "rpcgssd", + "rpcidmapd", + "rsyncd@", + "rsyslog", + "selinux-policy-migrate-local-changes@", + "sshd-keygen", + "sshd", + "sshd@", + "systemd-ask-password********", + "systemd-backlight@", + "systemd-binfmt", + "systemd-firstboot", + "systemd-fsck-root", + "systemd-fsck@", + "systemd-halt", + "systemd-hibernate-resume@", + "systemd-hibernate", + "systemd-hostnamed", + "systemd-hwdb-update", + "systemd-hybrid-sleep", + "systemd-importd", + "systemd-initctl", + "systemd-journal-catalog-update", + "systemd-journal-flush", + "systemd-journald", + "systemd-kexec", + "systemd-localed", + "systemd-logind", + "systemd-machine-id-commit", + "systemd-machined", + "systemd-modules-load", + "systemd-poweroff", + "systemd-quotacheck", + "systemd-random-seed", + "systemd-readahead-collect", + "systemd-readahead-done", + "systemd-readahead-drop", + "systemd-readahead-replay", + "systemd-reboot", + "systemd-remount-fs", + "systemd-rfkill@", + "systemd-shutdownd", + "systemd-suspend", + "systemd-sysctl", + "systemd-timedated", + "systemd-tmpfiles-clean", + "systemd-tmpfiles-setup-dev", + "systemd-tmpfiles-setup", + "systemd-udev-settle", + "systemd-udev-trigger", + "systemd-udevd", + "systemd-update-done", + "systemd-update-utmp-runlevel", + "systemd-update-utmp", + "systemd-user-sessions", + "systemd-vconsole-setup", + "teamd@", + "tuned" + ], + "infrastructure_type": "virtual", + "infrastructure_vendor": "kvm", + "insights_client_version": "3.0.9-1.el7", + "insights_egg_version": "3.0.133-1", + "installed_packages": "{{ installed_packages|tojson }}", + "installed_products": [ + { + "id": "230" + } + ], + "installed_services": [ + "arp-ethers", + "auditd", + "auth-rpcgss-module", + "autovt@", + "blk-availability", + "brandbot", + "chrony-dnssrv@", + "chrony-wait", + "chronyd", + "cloud-config", + "cloud-final", + "cloud-init-local", + "cloud-init", + "console-getty", + "console-shell", + "container-getty@", + "cpupower", + "crond", + "dbus-org.freedesktop.hostname1", + "dbus-org.freedesktop.import1", + "dbus-org.freedesktop.locale1", + "dbus-org.freedesktop.login1", + "dbus-org.freedesktop.machine1", + "dbus-org.freedesktop.nm-dispatcher", + "dbus-org.freedesktop.timedate1", + "dbus", + "debug-shell", + "dracut-cmdline", + "dracut-initqueue", + "dracut-mount", + "dracut-pre-mount", + "dracut-pre-pivot", + "dracut-pre-trigger", + "dracut-pre-udev", + "dracut-shutdown", + "emergency", + "fstrim", + "getty@", + "gssproxy", + "halt-local", + "initrd-cleanup", + "initrd-parse-etc", + "initrd-switch-root", + "initrd-udevadm-cleanup-db", + "insights-client", + "irqbalance", + "kdump", + "kmod-static-nodes", + "messagebus", + "microcode", + "NetworkManager-dispatcher", + "NetworkManager-wait-online", + "NetworkManager", + "nfs-blkmap", + "nfs-config", + "nfs-idmap", + "nfs-idmapd", + "nfs-lock", + "nfs-mountd", + "nfs-rquotad", + "nfs-secure", + "nfs-server", + "nfs-utils", + "nfs", + "nfslock", + "ovirt-guest-agent", + "polkit", + "postfix", + "qemu-guest-agent", + "quotaon", + "rc-local", + "rdisc", + "rescue", + "rhel-autorelabel-mark", + "rhel-autorelabel", + "rhel-configure", + "rhel-dmesg", + "rhel-domainname", + "rhel-import-state", + "rhel-loadmodules", + "rhel-readonly", + "rhsm-facts", + "rhsm", + "rhsmcertd", + "rpc-gssd", + "rpc-rquotad", + "rpc-statd-notify", + "rpc-statd", + "rpcbind", + "rpcgssd", + "rpcidmapd", + "rsyncd", + "rsyncd@", + "rsyslog", + "selinux-policy-migrate-local-changes@", + "serial-gettystemd-journal-catalog-update", + "systemd-journal-flush", + "systemd-journald", + "systemd-kexec", + "systemd-localed", + "systemd-logind", + "systemd-machine-id-commit", + "systemd-machined", + "systemd-modules-load", + "systemd-nspawn@", + "systemd-poweroff", + "systemd-quotacheck", + "systemd-random-seed", + "systemd-readahead-collect", + "systemd-readahead-done", + "systemd-readahead-drop", + "systemd-readahead-replay", + "systemd-reboot", + "systemd-remount-fs", + "systemd-rfkill@", + "systemd-shutdownd", + "systemd-suspend", + "systemd-sysctl", + "systemd-timedated", + "systemd-tmpfiles-clean", + "systemd-tmpfiles-setup-dev", + "systemd-tmpfiles-setup", + "systemd-udev-settle", + "systemd-udev-trigger", + "systemd-udevd", + "systemd-update-done", + "systemd-update-utmp-runlevel", + "systemd-update-utmp", + "systemd-user-sessions", + "systemd-vconsole-setup", + "teamd@", + "tuned", + "wpa_supplicant" + ], + "kernel_modules": [ + "dm_mod", + "binfmt_misc", + "virtio_console", + "nfit", + "libnvdimm", + "iosf_mbi", + "kvm_intel", + "kvm", + "irqbypass", + "crc32_pclmul", + "ghash_clmulni_intel", + "cirrus", + "ttm", + "drm_kms_helper", + "syscopyarea", + "sysfillrect", + "sysimgblt", + "fb_sys_fops", + "drm", + "aesni_intel", + "ppdev", + "lrw", + "gf128mul", + "glue_helper", + "ablk_helper", + "cryptd", + "parport_pc", + "pcspkr", + "drm_panel_orientation_quirks", + "joydev", + "virtio_balloon", + "i2c_piix4", + "parport", + "ip_tables", + "xfs", + "libcrc32c", + "ata_generic", + "pata_acpi", + "virtio_net", + "virtio_blk", + "net_failover", + "failover", + "ata_piix", + "libata", + "floppy", + "crct10dif_pclmul", + "crct10dif_common", + "crc32c_intel", + "serio_raw", + "virtio_pci", + "virtio_ring", + "virtio", + "sunrpc" + ], + "last_boot_time": "2019-11-19T21:47:22", + "network_interfaces": [ + { + "ipv4_addresses": [ + "127.0.0.1" + ], + "ipv6_addresses": [ + "::1" + ], + "mac_address": "00:00:00:00:00:00", + "mtu": 65536, + "name": "lo", + "state": "UNKNOWN", + "type": "loopback" + }, + { + "ipv4_addresses": [ + "{{ ipv4_addr }}" + ], + "ipv6_addresses": [ + "{{ ipv6_addr }}" + ], + "mac_address": "{{ mac_addr }}", + "mtu": 1500, + "name": "eth0", + "state": "UP", + "type": "ether" + } + ], + "number_of_cpus": 1, + "number_of_sockets": 1, + "os_kernel_version": "3.10.0", + "os_release": "7.5", + "running_processes": [ + "master", + "migration/0", + "scsi_tmf_0", + "su", + "anacron", + "insights-client", + "deferwq", + "sudo", + "bioset", + "kintegrityd", + "xfs-reclaim/vda", + "scsi_tmf_1", + "nfit", + "ata_sff", + "kworker/0:0H", + "systemd", + "xfsaild/vda1", + "scsi_eh_1", + "xfs-data/vda1", + "netns", + "scsi_eh_0", + "xfs-cil/vda1", + "rcu_sched", + "crypto", + "xfsalloc", + "systemd-logind", + "kblockd", + "tuned", + "qmgr", + "edac-poller", + "rhnsd", + "kaluad", + "xfs-log/vda1", + "kdevtmpfs", + "rsyslogd", + "ps", + "ttm_swap", + "kmpath_rdacd", + "lru-add-drain", + "xfs_mru_cache", + "kworker/u2:0", + "systemd-journal", + "kpsmoused", + "dbus-daemon", + "polkitd", + "chronyd", + "kauditd", + "pickup", + "ksmd", + "gssproxy", + "dhclient", + "bash", + "watchdog/0", + "sshd", + "kworker/0:0", + "kworker/0:2", + "khungtaskd", + "kthrotld", + "kworker/u2:1", + "rpcbind", + "xfs-buf/vda1", + "systemd-udevd", + "md", + "xprtiod", + "kthreadd", + "NetworkManager", + "ipv6_addrconf", + "khugepaged", + "writeback", + "crond", + "rpciod", + "kworker/0:1H", + "auditd", + "ksoftirqd/0", + "kworker/0:1", + "rcu_bh", + "kswapd0", + "xfs-eofblocks/v", + "timeout", + "watchdogd", + "agetty", + "xfs-conv/vda1", + "kvm-irqfd-clean" + ], + "satellite_managed": false, + "system_memory_bytes": 1927180288, + "yum_repos": "{{ yum_repos|tojson }}" + }, + "tags": [], + "updated": "{{ nowz }}" + }, + "platform_metadata": { + "account": "{{ account }}", + "b64_identity": "{{ b64_identity }}", + "category": "something", + "elapsed_time": 1586859092.4174569, + "extras": {}, + "host": {}, + "id": null, + "metadata": { + "reporter": "", + "stale_timestamp": "0001-01-01T00:00:00Z" + }, + "payload_id": null, + "principal": "5894300", + "request_id": "{{ request_id }}", + "satellite_managed": null, + "service": "advisor", + "size": 195324, + "test": null, + "timestamp": "{{ nowz }}", + "url": "https://XXXXXXXXXXXXX.s3.amazonaws.com/018c78be10954bf689974590c19da741?X", + "validation": null + }, + "timestamp": "{{ nowz }}", + "type": "{{ msg_type }}" +} diff --git a/DELME/opl/generators/testit.py b/DELME/opl/generators/testit.py new file mode 100644 index 00000000..fcdfc533 --- /dev/null +++ b/DELME/opl/generators/testit.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import opl.generators.playbook_dispatcher + +generator = opl.generators.playbook_dispatcher.RunnerUpdatesGenerator( + count=3, + template='playbook-dispatcher-runner-updates.json.j2', + correlation_id="123", + run_id="456", +) + +for msg_id, msg_struct in generator: + print(f"Message {msg_id}: {str(msg_struct)[:100]}") diff --git a/DELME/opl_dep_test b/DELME/opl_dep_test new file mode 160000 index 00000000..9a218c04 --- /dev/null +++ b/DELME/opl_dep_test @@ -0,0 +1 @@ +Subproject commit 9a218c0442f6f91bca41227954258584c3ee0b66 diff --git a/DELME/size_kb.py b/DELME/size_kb.py new file mode 100644 index 00000000..34bf7388 --- /dev/null +++ b/DELME/size_kb.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import sys +import random +import string + +""" +Kafka allows only up to 1mb size per topic, so we are transforming script to kb + +Script to generate random python dict of size 100kb, 300kb, 500kb +""" + + +# Function to generate random string +def generate_random_string(size): + return "".join(random.choices(string.ascii_letters + string.digits, k=size)) + + +# Function to generate a random dictionary of a given size (in bytes) +def generate_random_dict(size_bytes): + size_bytes -= sys.getsizeof(dict()) # Account for dictionary overhead + size_bytes -= sys.getsizeof("") * 2 # Account for two empty strings + d = {} + while sys.getsizeof(d) < size_bytes: + key = generate_random_string(random.randint(5, 20)) + value = generate_random_string(random.randint(10, 50)) + d[key] = value + return d + + +# Generate a random dictionary approximately 5kb in size +def size_of_dict(size=100): + size_5kb = size * 1000 # 5MB in bytes + random_dict = generate_random_dict(size_5kb) + # Print the size of the generated dictionary in bytes + return random_dict diff --git a/DELME/tables.yaml b/DELME/tables.yaml new file mode 100644 index 00000000..8ce8c12c --- /dev/null +++ b/DELME/tables.yaml @@ -0,0 +1,14 @@ +tables: + items: + - CREATE TABLE IF NOT EXISTS items ( + subscription_manager_id VARCHAR PRIMARY KEY, + qpc_at TIMESTAMP WITH TIME ZONE NULL, + created_at TIMESTAMP WITH TIME ZONE NULL) + - CREATE INDEX IF NOT EXISTS items_subscription_manager_id_idx + ON items (subscription_manager_id) +queries: + query_store_info_produced: INSERT INTO items(subscription_manager_id, qpc_at) VALUES %s + query_storage_update_created_at: UPDATE items SET created_at = data.created_at FROM (VALUES %s) AS data(subscription_manager_id, created_at) WHERE items.subscription_manager_id = data.subscription_manager_id + query_storage_count_applicable_hosts: SELECT COUNT(*) FROM items WHERE qpc_at IS NOT NULL AND created_at IS NULL + query_storage_get_applicable_hosts: SELECT subscription_manager_id FROM items WHERE qpc_at IS NOT NULL AND created_at IS NULL OFFSET %s LIMIT %s + query_edge_get_created_at: SELECT uuid, created_at FROM devices WHERE uuid=ANY(%s) diff --git a/DELME/test.py b/DELME/test.py new file mode 100644 index 00000000..7ad108e4 --- /dev/null +++ b/DELME/test.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import pprint + +import opl.generators.inventory_egress +import opl.generators.inventory_ingress +import opl.generators.generic + +generator = opl.generators.inventory_egress.EgressHostsGenerator(count=1,n_packages=10,) +#generator = opl.generators.generic.GenericGenerator(count=1, template='inventory_egress_template.json.j2') +#generator = opl.generators.generic.GenericGenerator(count=1, template='inventory_ingress_RHSM_template.json.j2') +#generator = opl.generators.inventory_ingress.InventoryIngressGenerator(count=1, fraction=1, relatives=100, addresses=3, packages=10, template='inventory_ingress_RHSM_template.json.j2') + +for msg_id, msg_struct in generator: + print(msg_id) + pprint.pprint(msg_struct) diff --git a/DELME/test2.py b/DELME/test2.py new file mode 100755 index 00000000..5d12ed49 --- /dev/null +++ b/DELME/test2.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import json +import socket + +import opl.generators.inventory_egress +import opl.post_kafka_times + + +def func_add_more_args(parser): + parser.add_argument( + "--count", + default=100, + type=int, + help="How many messages to prepare", + ) + parser.add_argument( + "--n-packages", + default=500, + type=int, + help="How many packages addresses should each host have", + ) + parser.add_argument( + "--msg-type", + default="created", + choices=["created"], + help="Type of the message", + ) + + +def func_return_generator(args): + return opl.generators.inventory_egress.EgressHostsGenerator( + count=args.count, + n_packages=args.n_packages, + msg_type=args.msg_type, + ) + + +def func_return_message_payload(args, message_id, message): + return json.dumps(message) + + +def func_return_message_key(args, message_id, message): + return message_id + + +def func_return_message_headers(args, message_id, message): + _event_type = args.msg_type + _request_id = message["platform_metadata"]["request_id"] + _producer = socket.gethostname() + _insights_id = message["host"]["insights_id"] + return [ + ("event_type", _event_type), + ("request_id", _request_id), + ("producer", _producer), + ("insights_id", _insights_id), + ] + + +if __name__ == "__main__": + config = { + "func_add_more_args": func_add_more_args, + "query_store_info_produced": "query_store_info_produced", + "func_return_generator": func_return_generator, + "func_return_message_payload": func_return_message_payload, + "func_return_message_headers": func_return_message_headers, + "func_return_message_key": func_return_message_key, + } + opl.post_kafka_times.post_kafka_times(config) diff --git a/DELME/test3.py b/DELME/test3.py new file mode 100755 index 00000000..a58b2d29 --- /dev/null +++ b/DELME/test3.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +import opl.args +import opl.get_db_times + + +def func_add_more_args(parser): + opl.args.add_edge_db_opts(parser) + + +def func_create_app_db_config(args): + return { + 'host': args.edge_db_host, + 'port': args.edge_db_port, + 'database': args.edge_db_name, + 'user': args.edge_db_user, + 'password': args.edge_db_pass, + } + + +if __name__ == "__main__": + config = { + "func_add_more_args": func_add_more_args, + "func_create_app_db_config": func_create_app_db_config, + "query_storage_update_timestamp": "query_storage_update_created_at", # update query to store created_at into items table + "query_storage_count_applicable_hosts": "query_storage_count_applicable_hosts", # query to get count of hosts with missing created_at value + "query_storage_get_applicable_hosts": "query_storage_get_applicable_hosts", # query to get IDs of hosts with missing created_at value + "query_app_get_hosts": "query_edge_get_created_at", # query host ID and created_at from Edge DB for given hosts + } + opl.get_db_times.get_db_times(config) diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-09T20_13_17_629_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-09T20_13_17_629_0000-chatty.json new file mode 100644 index 00000000..23bf3f6b --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-09T20_13_17_629_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-09T20_13_17_629_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-09T20:17:47.576361+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-09T20_57_10_214_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-09T20_57_10_214_0000-chatty.json new file mode 100644 index 00000000..f5cc23cd --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-09T20_57_10_214_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-09T20_57_10_214_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-09T21:00:47.040691+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-09T21_32_44_337_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-09T21_32_44_337_0000-chatty.json new file mode 100644 index 00000000..ca54f184 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-09T21_32_44_337_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-09T21_32_44_337_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-09T23:24:10.089107+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-api.json new file mode 100644 index 00000000..d8978675 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-10T11:39:32.053334+00:00", + "golden": "true", + "id": "run-2021-12-10T08_38_02_566_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2286.0999999999995, + "max": 26934.266666666666, + "mean": 5305.032034466417, + "median": 3254.5333333333333, + "min": 1437.9333333333334, + "non_zero_mean": 5305.032034466417, + "non_zero_median": 3254.5333333333333, + "percentile25": 2779.7, + "percentile75": 5065.799999999999, + "percentile90": 11655.373333333335, + "percentile99": 23349.877333333334, + "percentile999": 26198.430933333493, + "range": 25496.333333333332, + "samples": 263, + "stdev": 4827.688884189489, + "sum": 1395223.4250646662 + } + }, + "cpu": { + "system": { + "iqr": 0.19200000000000156, + "max": 1.7706666666666668, + "mean": 0.26967552919640597, + "median": 0.11799999999999879, + "min": 0.03466666666666546, + "non_zero_mean": 0.26967552919640597, + "non_zero_median": 0.11799999999999879, + "percentile25": 0.060000000000000386, + "percentile75": 0.25200000000000194, + "percentile90": 0.8762666666666644, + "percentile99": 1.6795066666666645, + "percentile999": 1.751802666666671, + "range": 1.7360000000000013, + "samples": 263, + "stdev": 0.38529658842517384, + "sum": 70.92466417865481 + }, + "user": { + "iqr": 0.5353333333333166, + "max": 6.25000000000003, + "mean": 1.3578960057577008, + "median": 0.7533333333333302, + "min": 0.41399999999998727, + "non_zero_mean": 1.3578960057577008, + "non_zero_median": 0.7533333333333302, + "percentile25": 0.5903333333333345, + "percentile75": 1.125666666666651, + "percentile90": 3.7529333333332797, + "percentile99": 6.170159999999978, + "percentile999": 6.235502666666689, + "range": 5.836000000000043, + "samples": 263, + "stdev": 1.4511170335501726, + "sum": 357.1266495142756 + } + }, + "entropy_available_bits": { + "iqr": 2.2, + "max": 213.06666666666666, + "mean": 20.745754017820587, + "median": 1.7999999999999998, + "min": 0.13333333333333333, + "non_zero_mean": 20.745754017820587, + "non_zero_median": 1.7999999999999998, + "percentile25": 0.6666666666666666, + "percentile75": 2.8666666666666667, + "percentile90": 110.70666666666813, + "percentile99": 211.69199999999998, + "percentile999": 213.03173333333334, + "range": 212.93333333333334, + "samples": 263, + "stdev": 56.93443993534054, + "sum": 5456.13330668682 + }, + "memory": { + "used": { + "iqr": 266254336.0, + "max": 16323670016.0, + "mean": 6490519528.638783, + "median": 6011228160.0, + "min": 5578518528.0, + "non_zero_mean": 6490519528.638783, + "non_zero_median": 6011228160.0, + "percentile25": 5902424064.0, + "percentile75": 6168678400.0, + "percentile90": 6961038950.4000025, + "percentile99": 14634212311.039999, + "percentile999": 15926212075.520086, + "range": 10745151488.0, + "samples": 263, + "stdev": 1704997538.1331966, + "sum": 1707006636032.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 28.066666666666666, + "mean": 0.19163498098859316, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6.3, + "non_zero_median": 1.1333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.621333333333299, + "percentile999": 23.333200000001018, + "range": 28.066666666666666, + "samples": 263, + "stdev": 1.9228762529694718, + "sum": 50.4 + } + }, + "writes_completed": { + "total": { + "iqr": 86.5, + "max": 1861.8666666666666, + "mean": 122.60987782110674, + "median": 29.6, + "min": 0.0, + "non_zero_mean": 124.9860382439964, + "non_zero_median": 31.366666666666667, + "percentile25": 1.4333333333333331, + "percentile75": 87.93333333333334, + "percentile90": 331.0533333333335, + "percentile99": 1239.1879999999994, + "percentile999": 1747.2154666666913, + "range": 1861.8666666666666, + "samples": 263, + "stdev": 267.9214728435774, + "sum": 32246.397866951076 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 989.3333333333333, + "max": 3105.866666666667, + "mean": 1613.3984790874524, + "median": 1493.8, + "min": 438.26666666666665, + "non_zero_mean": 1613.3984790874524, + "non_zero_median": 1493.8, + "percentile25": 1151.2333333333333, + "percentile75": 2140.5666666666666, + "percentile90": 2432.1466666666665, + "percentile99": 3020.5719999999997, + "percentile999": 3092.2077333333364, + "range": 2667.6000000000004, + "samples": 263, + "stdev": 590.9162787395878, + "sum": 424323.7999999997 + } + }, + "cpu": { + "system": { + "iqr": 0.04399999999999788, + "max": 0.2406666666666657, + "mean": 0.08567807351077313, + "median": 0.07400000000000091, + "min": 0.014000000000000531, + "non_zero_mean": 0.08567807351077313, + "non_zero_median": 0.07400000000000091, + "percentile25": 0.05600000000000022, + "percentile75": 0.0999999999999981, + "percentile90": 0.15520000000000098, + "percentile99": 0.22666666666666585, + "percentile999": 0.23979333333333283, + "range": 0.22666666666666516, + "samples": 263, + "stdev": 0.044502178483120505, + "sum": 22.533333333333328 + }, + "user": { + "iqr": 0.12200000000000272, + "max": 1.1146666666666647, + "mean": 0.2038555133079848, + "median": 0.19399999999999978, + "min": 0.01866666666666864, + "non_zero_mean": 0.2038555133079848, + "non_zero_median": 0.19399999999999978, + "percentile25": 0.13466666666666544, + "percentile75": 0.25666666666666815, + "percentile90": 0.32279999999999975, + "percentile99": 0.49909333333333367, + "percentile999": 0.9737106666666941, + "range": 1.095999999999996, + "samples": 263, + "stdev": 0.11041671892178197, + "sum": 53.614000000000004 + } + }, + "entropy_available_bits": { + "iqr": 1.3666666666666667, + "max": 212.4, + "mean": 11.246894803548797, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 20.259817351598173, + "non_zero_median": 1.3333333333333333, + "percentile25": 0.0, + "percentile75": 1.3666666666666667, + "percentile90": 2.8666666666666667, + "percentile99": 212.076, + "percentile999": 212.36506666666668, + "range": 212.4, + "samples": 263, + "stdev": 45.76914392345947, + "sum": 2957.9333333333348 + }, + "memory": { + "used": { + "iqr": 23076864.0, + "max": 1092739072.0, + "mean": 433257405.8098859, + "median": 392753152.0, + "min": 367079424.0, + "non_zero_mean": 433257405.8098859, + "non_zero_median": 392753152.0, + "percentile25": 384569344.0, + "percentile75": 407646208.0, + "percentile90": 527672934.4000004, + "percentile99": 1009003233.2799999, + "percentile999": 1085562904.5760016, + "range": 725659648.0, + "samples": 263, + "stdev": 127022913.23693186, + "sum": 113946697728.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 5461.333333333333, + "max": 59577197.8, + "mean": 1341144.9089987325, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3636300.114089347, + "non_zero_median": 12014.933333333332, + "percentile25": 0.0, + "percentile75": 5461.333333333333, + "percentile90": 33860.2666666667, + "percentile99": 57800778.984, + "percentile999": 59492347.24853335, + "range": 59577197.8, + "samples": 263, + "stdev": 8719008.75573662, + "sum": 352721111.0666666 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 364.10928, + "mean": 3.3063036577946767, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 24.84451034285714, + "non_zero_median": 0.037409, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0037728000000000128, + "percentile99": 58.75936031999931, + "percentile999": 360.18314528600087, + "range": 364.10928, + "samples": 263, + "stdev": 32.39920835537902, + "sum": 869.5578620000001 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.3192605, + "max": 424.122361, + "mean": 24.0898745095057, + "median": 0.0, + "min": -0.000727, + "non_zero_mean": 56.06758403539823, + "non_zero_median": 1.282774, + "percentile25": 0.0, + "percentile75": 0.3192605, + "percentile90": 75.29128160000016, + "percentile99": 384.8224652199999, + "percentile999": 420.1894129760009, + "range": 424.123088, + "samples": 263, + "stdev": 74.38458927291518, + "sum": 6335.636996000002 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 5.6, + "mean": 0.13992395437262356, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.830769230769231, + "non_zero_median": 2.933333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.3586666666666662, + "percentile999": 5.198266666666753, + "range": 5.6, + "samples": 263, + "stdev": 0.6627548036292021, + "sum": 36.8 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9993.4, + "mean": 433.0390367553866, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8760.712820512821, + "non_zero_median": 9006.666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9951.270666666667, + "percentile999": 9992.736266666667, + "range": 9993.4, + "samples": 263, + "stdev": 1920.43184091586, + "sum": 113889.26666666666 + }, + "pg_stat_database_blks_hit": { + "iqr": 21422.966666666667, + "max": 195437.73333333334, + "mean": 25666.03041825095, + "median": 26022.666666666668, + "min": 2476.4, + "non_zero_mean": 25666.03041825095, + "non_zero_median": 26022.666666666668, + "percentile25": 11773.033333333333, + "percentile75": 33196.0, + "percentile90": 44220.77333333334, + "percentile99": 61634.289333333254, + "percentile999": 187038.0308000018, + "range": 192961.33333333334, + "samples": 263, + "stdev": 19752.971176184605, + "sum": 6750166.0 + }, + "pg_stat_database_blks_read": { + "iqr": 0.6333333333333333, + "max": 215.53333333333333, + "mean": 1.421039290240811, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7967948717948719, + "non_zero_median": 0.4, + "percentile25": 0.06666666666666667, + "percentile75": 0.7, + "percentile90": 2.066666666666667, + "percentile99": 3.933333333333333, + "percentile999": 160.89760000001175, + "range": 215.53333333333333, + "samples": 263, + "stdev": 13.28704739970201, + "sum": 373.7333333333331 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 263, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.6333333333333333, + "max": 215.53333333333333, + "mean": 1.421039290240811, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7967948717948719, + "non_zero_median": 0.4, + "percentile25": 0.06666666666666667, + "percentile75": 0.7, + "percentile90": 2.066666666666667, + "percentile99": 3.933333333333333, + "percentile999": 160.89760000001175, + "range": 215.53333333333333, + "samples": 263, + "stdev": 13.28704739970201, + "sum": 373.7333333333331 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.8666666666666667, + "max": 4005.733333333333, + "mean": 41.25373891001267, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 101.3993769470405, + "non_zero_median": 14.2, + "percentile25": 0.0, + "percentile75": 2.8666666666666667, + "percentile90": 56.93333333333334, + "percentile99": 88.964, + "percentile999": 4005.052133333333, + "range": 4005.733333333333, + "samples": 263, + "stdev": 348.311208694115, + "sum": 10849.733333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11381.533333333335, + "max": 321483.0, + "mean": 14378.969328263625, + "median": 13980.266666666666, + "min": 1395.6666666666667, + "non_zero_mean": 14378.969328263625, + "non_zero_median": 13980.266666666666, + "percentile25": 5730.866666666667, + "percentile75": 17112.4, + "percentile90": 24537.213333333337, + "percentile99": 37836.05066666666, + "percentile999": 247540.13733334924, + "range": 320087.3333333333, + "samples": 263, + "stdev": 20752.935648121023, + "sum": 3781668.933333335 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.5, + "max": 8000.066666666667, + "mean": 46.177946768060835, + "median": 2.533333333333333, + "min": 0.0, + "non_zero_mean": 54.95384615384615, + "non_zero_median": 3.8, + "percentile25": 1.2, + "percentile75": 9.7, + "percentile90": 65.6400000000001, + "percentile99": 145.93333333333334, + "percentile999": 5942.807733333776, + "range": 8000.066666666667, + "samples": 263, + "stdev": 493.41240013910806, + "sum": 12144.799999999997 + }, + "pg_stat_database_tup_returned": { + "iqr": 30790.73333333333, + "max": 557880.6, + "mean": 43545.73257287706, + "median": 35372.46666666667, + "min": 3305.4666666666667, + "non_zero_mean": 43545.73257287706, + "non_zero_median": 35372.46666666667, + "percentile25": 19759.800000000003, + "percentile75": 50550.53333333333, + "percentile90": 82595.93333333333, + "percentile99": 168236.85066666667, + "percentile999": 502757.58160001185, + "range": 554575.1333333333, + "samples": 263, + "stdev": 48765.93513200842, + "sum": 11452527.66666668 + }, + "pg_stat_database_tup_updated": { + "iqr": 13.466666666666667, + "max": 66.0, + "mean": 10.347528517110266, + "median": 7.266666666666667, + "min": 0.13333333333333333, + "non_zero_mean": 10.347528517110266, + "non_zero_median": 7.266666666666667, + "percentile25": 3.466666666666667, + "percentile75": 16.933333333333334, + "percentile90": 22.986666666666668, + "percentile99": 39.536, + "percentile999": 61.72066666666759, + "range": 65.86666666666666, + "samples": 263, + "stdev": 9.681778199803421, + "sum": 2721.3999999999987 + }, + "pg_stat_database_xact_commit": { + "iqr": 125.66666666666666, + "max": 704.6, + "mean": 122.85221799746515, + "median": 91.53333333333333, + "min": 4.333333333333333, + "non_zero_mean": 122.85221799746515, + "non_zero_median": 91.53333333333333, + "percentile25": 39.9, + "percentile75": 165.56666666666666, + "percentile90": 261.84000000000003, + "percentile99": 568.2639999999999, + "percentile999": 702.8708000000004, + "range": 700.2666666666667, + "samples": 263, + "stdev": 124.49233205461589, + "sum": 32310.133333333328 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.12648922686945502, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.12648922686945502, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 263, + "stdev": 0.14481295220412826, + "sum": 33.26666666666667 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.8666666666666667, + "mean": 0.007097591888466414, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.8666666666666667, + "non_zero_median": 1.8666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 1.3776000000001054, + "range": 1.8666666666666667, + "samples": 263, + "stdev": 0.11510359764926538, + "sum": 1.8666666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 22.83333333333334, + "max": 142.0, + "mean": 38.992648922686946, + "median": 31.733333333333334, + "min": 1.7333333333333334, + "non_zero_mean": 38.992648922686946, + "non_zero_median": 31.733333333333334, + "percentile25": 23.333333333333332, + "percentile75": 46.16666666666667, + "percentile90": 71.52, + "percentile99": 115.95599999999999, + "percentile999": 136.89973333333444, + "range": 140.26666666666668, + "samples": 263, + "stdev": 23.52880024757826, + "sum": 10255.06666666667 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-10T10:33:45.795171+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-10T08_38_02_566_0000", + "ended": "2021-12-10T11:39:27.162324+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.01597153424950193, + "iterations": 1, + "max": 0.5878419999999096, + "mean": 0.5549973437333998, + "median": 0.5537240040002871, + "min": 0.5252806899998177, + "rounds": 15, + "stddev": 0.013996976993116266 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.012769572249681005, + "iterations": 1, + "max": 0.5816970630003198, + "mean": 0.551331964133351, + "median": 0.5519270949998827, + "min": 0.5311693220000961, + "rounds": 15, + "stddev": 0.01350797611964908 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.016557534000639862, + "iterations": 1, + "max": 0.47843667500001175, + "mean": 0.4076271161001387, + "median": 0.4049697069999638, + "min": 0.3749281440004779, + "rounds": 10, + "stddev": 0.02786077916128598 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.006363670499922591, + "iterations": 1, + "max": 1.0266870429995834, + "mean": 0.26900758191998647, + "median": 0.2591550840002128, + "min": 0.2443278279997685, + "rounds": 100, + "stddev": 0.07781983041768399 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.014482697999483207, + "iterations": 1, + "max": 0.6412695640001402, + "mean": 0.4798511775499537, + "median": 0.4784979060000296, + "min": 0.44771602299988444, + "rounds": 100, + "stddev": 0.021705330591397502 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.10347473899946635, + "iterations": 1, + "max": 1.4734625090004556, + "mean": 1.3463106266000069, + "median": 1.3270971370002371, + "min": 1.2612620969994168, + "rounds": 10, + "stddev": 0.06836541121543332 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.002199276999817812, + "iterations": 1, + "max": 0.0802420969994273, + "mean": 0.07411399714283107, + "median": 0.07396509850059374, + "min": 0.07109519499954331, + "rounds": 14, + "stddev": 0.0022485713627228635 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.00192350850056755, + "iterations": 1, + "max": 0.08511447100045189, + "mean": 0.07313460239996251, + "median": 0.072033360000205, + "min": 0.0694616450000467, + "rounds": 15, + "stddev": 0.0036839712885793872 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.01705721899998025, + "iterations": 1, + "max": 0.12460803399972065, + "mean": 0.10359213589999854, + "median": 0.09688926199987691, + "min": 0.093954314000257, + "rounds": 10, + "stddev": 0.01077241770942627 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.007390551000753476, + "iterations": 1, + "max": 0.11883966700042947, + "mean": 0.11104304689997661, + "median": 0.11064099399982297, + "min": 0.10536139299983915, + "rounds": 10, + "stddev": 0.004213593776479835 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.007147113000428362, + "iterations": 1, + "max": 0.284760217999974, + "mean": 0.27232744840011947, + "median": 0.2726010710002811, + "min": 0.2637299580001127, + "rounds": 10, + "stddev": 0.006210728383199888 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.008804219999547058, + "iterations": 1, + "max": 0.23863330299991503, + "mean": 0.1301189685001191, + "median": 0.11483945950021734, + "min": 0.10999472500043339, + "rounds": 10, + "stddev": 0.03996070904004329 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0022284569995463244, + "iterations": 1, + "max": 0.07550914599960379, + "mean": 0.07369250978568484, + "median": 0.07355690050007979, + "min": 0.07103305800046655, + "rounds": 14, + "stddev": 0.0013684059041116881 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.11946861400065245, + "iterations": 1, + "max": 0.7802562289998605, + "mean": 0.6735191268998278, + "median": 0.6425800454999262, + "min": 0.6093276849996982, + "rounds": 10, + "stddev": 0.0675457314677572 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0031353740002941777, + "iterations": 1, + "max": 0.10414841900001193, + "mean": 0.09763190009090811, + "median": 0.09705114400003367, + "min": 0.09313092400043388, + "rounds": 11, + "stddev": 0.0030768988819477074 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.003088150000621681, + "iterations": 1, + "max": 0.11504695400071796, + "mean": 0.10684602260007522, + "median": 0.10607919799986121, + "min": 0.10212294600023597, + "rounds": 10, + "stddev": 0.0035865294984784475 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.007047271000374167, + "iterations": 1, + "max": 0.1817473400005838, + "mean": 0.17730047309978544, + "median": 0.17827892399964185, + "min": 0.17229163299998618, + "rounds": 10, + "stddev": 0.0034746500308660455 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.002406492999398324, + "iterations": 1, + "max": 0.08852257600028679, + "mean": 0.07559279864283651, + "median": 0.07383723000020836, + "min": 0.07148703500024567, + "rounds": 14, + "stddev": 0.004907401866467904 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.004568308999978399, + "iterations": 1, + "max": 0.08967607300019154, + "mean": 0.07722928953829139, + "median": 0.0767189039997902, + "min": 0.07278956200025277, + "rounds": 13, + "stddev": 0.004478858484725098 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.009821615000873862, + "iterations": 1, + "max": 0.4116237979997095, + "mean": 0.38557241990001784, + "median": 0.3827379855001709, + "min": 0.3756223569998838, + "rounds": 10, + "stddev": 0.010423091598143574 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.029215646999546152, + "iterations": 1, + "max": 0.4067374590003965, + "mean": 0.38416322860002766, + "median": 0.3866760989994873, + "min": 0.35845074899953033, + "rounds": 10, + "stddev": 0.015391290160024771 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.0027678364992880233, + "iterations": 1, + "max": 0.10046078899995337, + "mean": 0.09694125445457757, + "median": 0.09693498100023135, + "min": 0.09243124600016017, + "rounds": 11, + "stddev": 0.0024872715000909167 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.002663212750121602, + "iterations": 1, + "max": 0.09600547899935918, + "mean": 0.09139136518181411, + "median": 0.09072533100061264, + "min": 0.0889891759998136, + "rounds": 11, + "stddev": 0.002024483557957684 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.007819710998774099, + "iterations": 1, + "max": 0.20130601999971987, + "mean": 0.18236197229989556, + "median": 0.1801495115000762, + "min": 0.17386531099964486, + "rounds": 10, + "stddev": 0.008441201370451004 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.002039523999883386, + "iterations": 1, + "max": 0.0750516919997608, + "mean": 0.07176367671432413, + "median": 0.07149855750003553, + "min": 0.06876653600011196, + "rounds": 14, + "stddev": 0.0014591205595575515 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.010612295999635535, + "iterations": 1, + "max": 0.2213997239996388, + "mean": 0.2134887179001453, + "median": 0.214897810000366, + "min": 0.2042862119997153, + "rounds": 10, + "stddev": 0.005922670174890263 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0026161857497299934, + "iterations": 1, + "max": 0.08510551400013355, + "mean": 0.08067447853857, + "median": 0.0808337999997093, + "min": 0.07642587700047443, + "rounds": 13, + "stddev": 0.002583343686071819 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.0075852650006709155, + "iterations": 1, + "max": 0.3636986679994152, + "mean": 0.35082306820004305, + "median": 0.3497027850003178, + "min": 0.3421532880001905, + "rounds": 10, + "stddev": 0.00694705698430874 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.008467975999337796, + "iterations": 1, + "max": 0.25352440899951034, + "mean": 0.24036463929987803, + "median": 0.24095868149970556, + "min": 0.22644762999971135, + "rounds": 10, + "stddev": 0.007471960023164592 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.006121205999988888, + "iterations": 1, + "max": 0.1695427650001875, + "mean": 0.16509230940018824, + "median": 0.16542581800013068, + "min": 0.16033581500050786, + "rounds": 10, + "stddev": 0.003316225267814343 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.12530664699988847, + "iterations": 1, + "max": 0.2390110400001504, + "mean": 0.1852897637001661, + "median": 0.2313521430005494, + "min": 0.1067107250000845, + "rounds": 10, + "stddev": 0.06335152771999132 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.0016919687498102576, + "iterations": 1, + "max": 0.08650198499981343, + "mean": 0.07291775946669077, + "median": 0.07213500900070358, + "min": 0.0681777849995342, + "rounds": 15, + "stddev": 0.004021046404581975 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.004359809500101619, + "iterations": 1, + "max": 0.10112323199973616, + "mean": 0.0975784891816974, + "median": 0.09889744499923836, + "min": 0.09135208199950284, + "rounds": 11, + "stddev": 0.0030482014177710248 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.002305875999809359, + "iterations": 1, + "max": 0.08025842600000033, + "mean": 0.07540786878566255, + "median": 0.07528687500052911, + "min": 0.0713661179997871, + "rounds": 14, + "stddev": 0.0024269273831936345 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.008200847000807698, + "iterations": 1, + "max": 0.13621308099936869, + "mean": 0.12572379950015602, + "median": 0.1265531704998466, + "min": 0.11630380700080423, + "rounds": 10, + "stddev": 0.0058305333030620144 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0024013500005821697, + "iterations": 1, + "max": 0.09871065799961798, + "mean": 0.07812346692844585, + "median": 0.07611182700020436, + "min": 0.07414858799984358, + "rounds": 14, + "stddev": 0.0064931768257294985 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.01157349500044802, + "iterations": 1, + "max": 0.6488959870002873, + "mean": 0.6278355015000671, + "median": 0.62643983199996, + "min": 0.6150205929998265, + "rounds": 10, + "stddev": 0.01011257970432094 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.03098965900062467, + "iterations": 1, + "max": 0.778407533999598, + "mean": 0.6440149401999407, + "median": 0.6360158820002653, + "min": 0.6051360719993681, + "rounds": 10, + "stddev": 0.05038686611688875 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.0046426210001300205, + "iterations": 1, + "max": 0.2975929200001701, + "mean": 0.28711620990015946, + "median": 0.2874824345003617, + "min": 0.27670391199990263, + "rounds": 10, + "stddev": 0.005666571832339472 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0020224170011715614, + "iterations": 1, + "max": 0.0753465520001555, + "mean": 0.07300262514302501, + "median": 0.07346007700016344, + "min": 0.0699588470006347, + "rounds": 14, + "stddev": 0.0014477876373871685 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0022798939999120194, + "iterations": 1, + "max": 0.18100090100051602, + "mean": 0.08222226850005297, + "median": 0.0743550125002912, + "min": 0.07223092500044004, + "rounds": 14, + "stddev": 0.028469845039765494 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0010829502493834298, + "iterations": 1, + "max": 0.08137381200049276, + "mean": 0.07666470461544332, + "median": 0.07644337500005349, + "min": 0.0739883620008186, + "rounds": 13, + "stddev": 0.0018299279750939204 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0013200329995015636, + "iterations": 1, + "max": 0.07849461400019209, + "mean": 0.07498126057141365, + "median": 0.07462287249973087, + "min": 0.07168955999986792, + "rounds": 14, + "stddev": 0.0017836626594107177 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.004122995000216179, + "iterations": 1, + "max": 0.30623532500067085, + "mean": 0.29429862189990674, + "median": 0.2934333099997275, + "min": 0.2890417799999341, + "rounds": 10, + "stddev": 0.004830306656562005 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.010082020000481862, + "iterations": 1, + "max": 0.24768728600065515, + "mean": 0.23290566550012953, + "median": 0.2307965680001871, + "min": 0.22270502500032308, + "rounds": 10, + "stddev": 0.007219145869040551 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.015288872999008163, + "iterations": 1, + "max": 1.0824114590004683, + "mean": 0.8766200550999201, + "median": 0.927375118999862, + "min": 0.5827408689992808, + "rounds": 10, + "stddev": 0.15991716076619072 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.19319376299881696, + "iterations": 1, + "max": 8.898345888999756, + "mean": 8.725959564299956, + "median": 8.695081608500459, + "min": 8.615339338999547, + "rounds": 10, + "stddev": 0.10089105801757023 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.033517529999699036, + "iterations": 1, + "max": 1.145040088999849, + "mean": 1.0334975261001091, + "median": 1.0218715385003634, + "min": 0.9923800679998749, + "rounds": 10, + "stddev": 0.04290183664828072 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.0897082790006607, + "iterations": 1, + "max": 9.630752384999141, + "mean": 9.016243179099911, + "median": 8.944138593999924, + "min": 8.859590717999708, + "rounds": 10, + "stddev": 0.22331638303887522 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.4567225660002805, + "iterations": 1, + "max": 1.5772201279996807, + "mean": 1.210166673799904, + "median": 1.1201484485000037, + "min": 1.0109242009993977, + "rounds": 10, + "stddev": 0.22204517732123039 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.012012952000077348, + "iterations": 1, + "max": 0.42191682400061836, + "mean": 0.4062008078999497, + "median": 0.4059897999995883, + "min": 0.37788412700047047, + "rounds": 10, + "stddev": 0.01277549497612456 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.09750908499881916, + "iterations": 1, + "max": 11.954021908999493, + "mean": 11.545576756899754, + "median": 11.48367147999943, + "min": 11.437895601000491, + "rounds": 10, + "stddev": 0.15537608118895 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.013899965000746306, + "iterations": 1, + "max": 0.4516791240002931, + "mean": 0.4096973174001505, + "median": 0.4100938325000243, + "min": 0.3892587659993296, + "rounds": 10, + "stddev": 0.017203919475166248 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.25068173900126567, + "iterations": 1, + "max": 0.8883759580003243, + "mean": 0.5928575012001602, + "median": 0.5722763554999801, + "min": 0.4244894240000576, + "rounds": 10, + "stddev": 0.1495011389939882 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.21173087499846588, + "iterations": 1, + "max": 11.954184509999322, + "mean": 11.71862962530031, + "median": 11.659084734001226, + "min": 11.572807920001651, + "rounds": 10, + "stddev": 0.12896661764393816 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.13616991599974426, + "iterations": 1, + "max": 8.038788315000602, + "mean": 7.733434853400103, + "median": 7.714665574000264, + "min": 7.5848663389997455, + "rounds": 10, + "stddev": 0.12936607445921067 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 269.5122835150014, + "mean": 269.5122835150014, + "median": 269.5122835150014, + "min": 269.5122835150014, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 399.75963188900096, + "mean": 399.75963188900096, + "median": 399.75963188900096, + "min": 399.75963188900096, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.011036740001145517, + "iterations": 1, + "max": 0.2719777800002703, + "mean": 0.2641621604998363, + "median": 0.26580555999953503, + "min": 0.2525149370003419, + "rounds": 10, + "stddev": 0.006623131674745357 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.012011537000034878, + "iterations": 1, + "max": 0.6605416430002151, + "mean": 0.6468892003000292, + "median": 0.6460664615001406, + "min": 0.6372805230002996, + "rounds": 10, + "stddev": 0.00794293308918704 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.2445534630005568, + "iterations": 1, + "max": 9.718152543000542, + "mean": 9.574738718400113, + "median": 9.623209438999766, + "min": 9.313122817000476, + "rounds": 10, + "stddev": 0.13704067181033158 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.866580428000816, + "iterations": 1, + "max": 7.282501285999388, + "mean": 3.2844042117000756, + "median": 2.7873465950001446, + "min": 2.2063062560000617, + "rounds": 10, + "stddev": 1.473672104980627 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.7885288819998095, + "iterations": 1, + "max": 3.3810739500004274, + "mean": 2.0644051146000493, + "median": 1.9674343679998856, + "min": 1.4947824540004149, + "rounds": 10, + "stddev": 0.5566918917716065 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 1.021014711001044, + "iterations": 1, + "max": 25.94262192199858, + "mean": 24.663702694399582, + "median": 24.517807117999837, + "min": 23.80161674199917, + "rounds": 10, + "stddev": 0.6602937067296433 + } + }, + "started": "2021-12-10T10:23:45.084992+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-chatty.json new file mode 100644 index 00000000..cb4ec5cb --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-10T09:35:23.567045+00:00", + "golden": "true", + "id": "run-2021-12-10T08_38_02_566_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 8115.850000000002, + "max": 43576.86666666667, + "mean": 32277.483333333334, + "median": 38162.86666666667, + "min": 1698.4, + "non_zero_mean": 32277.483333333334, + "non_zero_median": 38162.86666666667, + "percentile25": 31518.966666666664, + "percentile75": 39634.816666666666, + "percentile90": 41433.49333333333, + "percentile99": 43089.75066666667, + "percentile999": 43528.15506666667, + "range": 41878.46666666667, + "samples": 64, + "stdev": 12436.528514327847, + "sum": 2065758.9333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.14650000000000174, + "max": 1.0673333333333337, + "mean": 0.8546979166666666, + "median": 1.0059999999999996, + "min": 0.03666666666666695, + "non_zero_mean": 0.8546979166666666, + "non_zero_median": 1.0059999999999996, + "percentile25": 0.8789999999999994, + "percentile75": 1.0255000000000012, + "percentile90": 1.0423999999999976, + "percentile99": 1.0648133333333336, + "percentile999": 1.0670813333333338, + "range": 1.0306666666666668, + "samples": 64, + "stdev": 0.29262257395703706, + "sum": 54.70066666666668 + }, + "user": { + "iqr": 2.360833333333332, + "max": 6.522000000000003, + "mean": 5.1081562499999995, + "median": 6.313999999999999, + "min": 0.22399999999999523, + "non_zero_mean": 5.1081562499999995, + "non_zero_median": 6.313999999999999, + "percentile25": 4.015000000000008, + "percentile75": 6.37583333333334, + "percentile90": 6.419133333333325, + "percentile99": 6.50562, + "percentile999": 6.520362000000003, + "range": 6.298000000000008, + "samples": 64, + "stdev": 1.9533352104373218, + "sum": 326.9220000000001 + } + }, + "entropy_available_bits": { + "iqr": 7.966666666666667, + "max": 217.66666666666666, + "mean": 23.207291666666666, + "median": 7.1, + "min": 0.0, + "non_zero_mean": 32.28840579710145, + "non_zero_median": 7.699999999999999, + "percentile25": 0.0, + "percentile75": 7.966666666666667, + "percentile90": 17.39333333333334, + "percentile99": 214.39066666666665, + "percentile999": 217.3390666666667, + "range": 217.66666666666666, + "samples": 64, + "stdev": 57.33027076499513, + "sum": 1485.2666666666662 + }, + "memory": { + "used": { + "iqr": 1470632960.0, + "max": 7529213952.0, + "mean": 6348688384.0, + "median": 6733504512.0, + "min": 3452628992.0, + "non_zero_mean": 6348688384.0, + "non_zero_median": 6733504512.0, + "percentile25": 5608685568.0, + "percentile75": 7079318528.0, + "percentile90": 7200104038.4, + "percentile99": 7484602613.76, + "percentile999": 7524752818.176001, + "range": 4076584960.0, + "samples": 64, + "stdev": 934329694.984232, + "sum": 406316056576.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.7333333333333334, + "mean": 0.03333333333333333, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7111111111111111, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.8093333333333297, + "percentile999": 1.6409333333333402, + "range": 1.7333333333333334, + "samples": 64, + "stdev": 0.21902461346870272, + "sum": 2.1333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 2373.3666666666663, + "max": 4859.0, + "mean": 3520.5197916666666, + "median": 4566.333333333334, + "min": 2.8666666666666667, + "non_zero_mean": 3520.5197916666666, + "non_zero_median": 4566.333333333334, + "percentile25": 2301.0499999999997, + "percentile75": 4674.416666666666, + "percentile90": 4756.193333333334, + "percentile99": 4850.642, + "percentile999": 4858.1642, + "range": 4856.133333333333, + "samples": 64, + "stdev": 1622.8425565473224, + "sum": 225313.26666666666 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 227.6500000000001, + "max": 2226.6, + "mean": 1641.3, + "median": 1698.7333333333333, + "min": 592.4666666666667, + "non_zero_mean": 1641.3, + "non_zero_median": 1698.7333333333333, + "percentile25": 1557.5666666666666, + "percentile75": 1785.2166666666667, + "percentile90": 1970.0, + "percentile99": 2196.738, + "percentile999": 2223.6138, + "range": 1634.1333333333332, + "samples": 64, + "stdev": 317.12469264917024, + "sum": 105043.20000000003 + } + }, + "cpu": { + "system": { + "iqr": 0.028999999999999942, + "max": 0.25333333333333313, + "mean": 0.14561458333333333, + "median": 0.15466666666666667, + "min": 0.021999999999999888, + "non_zero_mean": 0.14561458333333333, + "non_zero_median": 0.15466666666666667, + "percentile25": 0.13683333333333317, + "percentile75": 0.1658333333333331, + "percentile90": 0.18706666666666694, + "percentile99": 0.24199333333333312, + "percentile999": 0.2521993333333332, + "range": 0.23133333333333325, + "samples": 64, + "stdev": 0.04492279164293676, + "sum": 9.319333333333327 + }, + "user": { + "iqr": 0.03949999999999981, + "max": 0.5566666666666668, + "mean": 0.22241666666666668, + "median": 0.18066666666666625, + "min": 0.029999999999999714, + "non_zero_mean": 0.22241666666666668, + "non_zero_median": 0.18066666666666625, + "percentile25": 0.16483333333333372, + "percentile75": 0.20433333333333353, + "percentile90": 0.4951333333333332, + "percentile99": 0.5545666666666669, + "percentile999": 0.5564566666666668, + "range": 0.5266666666666671, + "samples": 64, + "stdev": 0.13888071087034307, + "sum": 14.234666666666662 + } + }, + "entropy_available_bits": { + "iqr": 1.4833333333333334, + "max": 187.33333333333334, + "mean": 9.846875, + "median": 2.5, + "min": 0.5333333333333333, + "non_zero_mean": 9.846875, + "non_zero_median": 2.5, + "percentile25": 1.4, + "percentile75": 2.8833333333333333, + "percentile90": 4.600000000000002, + "percentile99": 175.4053333333333, + "percentile999": 186.14053333333342, + "range": 186.8, + "samples": 64, + "stdev": 34.613130578171, + "sum": 630.2000000000002 + }, + "memory": { + "used": { + "iqr": 26411008.0, + "max": 475856896.0, + "mean": 441818752.0, + "median": 453040128.0, + "min": 360464384.0, + "non_zero_mean": 441818752.0, + "non_zero_median": 453040128.0, + "percentile25": 434542592.0, + "percentile75": 460953600.0, + "percentile90": 467070156.8, + "percentile99": 474837606.4, + "percentile999": 475754967.04, + "range": 115392512.0, + "samples": 64, + "stdev": 30238699.232287172, + "sum": 28276400128.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1515929.5999999999, + "max": 3425348.2666666666, + "mean": 785800.5333333333, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2095468.088888889, + "non_zero_median": 2199005.8666666667, + "percentile25": 0.0, + "percentile75": 1515929.5999999999, + "percentile90": 2990462.2933333335, + "percentile99": 3410897.5786666665, + "percentile999": 3423903.1978666666, + "range": 3425348.2666666666, + "samples": 64, + "stdev": 1196602.5675731958, + "sum": 50291234.13333331 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.136127, + "mean": 0.00347621875, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.022247799999999998, + "non_zero_median": 0.0086985, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.006212600000000014, + "percentile99": 0.0641545399999997, + "percentile999": 0.12892975400000053, + "range": 0.136127, + "samples": 64, + "stdev": 0.017368641551607146, + "sum": 0.22247799999999998 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 0.276357, + "mean": 0.00740946875, + "median": 0.0, + "min": -0.000178, + "non_zero_mean": 0.027894470588235295, + "non_zero_median": 0.005279, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.008522400000000005, + "percentile99": 0.16394609999999954, + "percentile999": 0.26511591000000084, + "range": 0.27653500000000003, + "samples": 64, + "stdev": 0.036633508651588265, + "sum": 0.474206 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 19.6, + "mean": 0.3947916666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 12.633333333333335, + "non_zero_median": 12.633333333333335, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 10.821999999999964, + "percentile999": 18.72220000000007, + "range": 19.6, + "samples": 64, + "stdev": 2.53951639461503, + "sum": 25.26666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9984.0, + "mean": 311.63541666666663, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9972.333333333332, + "non_zero_median": 9972.333333333332, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9969.3, + "percentile999": 9982.53, + "range": 9984.0, + "samples": 64, + "stdev": 1748.8303208348798, + "sum": 19944.666666666664 + }, + "pg_stat_database_blks_hit": { + "iqr": 3238.866666666665, + "max": 39131.4, + "mean": 22015.334375, + "median": 22943.533333333333, + "min": 5342.2, + "non_zero_mean": 22015.334375, + "non_zero_median": 22943.533333333333, + "percentile25": 21143.05, + "percentile75": 24381.916666666664, + "percentile90": 26159.79333333333, + "percentile99": 38020.878, + "percentile999": 39020.34780000001, + "range": 33789.200000000004, + "samples": 64, + "stdev": 5823.537917661984, + "sum": 1408981.3999999997 + }, + "pg_stat_database_blks_read": { + "iqr": 68.24999999999999, + "max": 146.93333333333334, + "mean": 100.025, + "median": 130.73333333333335, + "min": 0.0, + "non_zero_mean": 101.6126984126984, + "non_zero_median": 130.86666666666667, + "percentile25": 66.63333333333334, + "percentile75": 134.88333333333333, + "percentile90": 141.38, + "percentile99": 146.30333333333334, + "percentile999": 146.87033333333335, + "range": 146.93333333333334, + "samples": 64, + "stdev": 49.05605115786249, + "sum": 6401.5999999999985 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 68.24999999999999, + "max": 146.93333333333334, + "mean": 100.025, + "median": 130.73333333333335, + "min": 0.0, + "non_zero_mean": 101.6126984126984, + "non_zero_median": 130.86666666666667, + "percentile25": 66.63333333333334, + "percentile75": 134.88333333333333, + "percentile90": 141.38, + "percentile99": 146.30333333333334, + "percentile999": 146.87033333333335, + "range": 146.93333333333334, + "samples": 64, + "stdev": 49.05605115786249, + "sum": 6401.5999999999985 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.03333333333333333, + "max": 1.4, + "mean": 0.2, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.03333333333333333, + "percentile90": 0.9800000000000002, + "percentile99": 1.3159999999999996, + "percentile999": 1.3916000000000006, + "range": 1.4, + "samples": 64, + "stdev": 0.3968126982095173, + "sum": 12.8 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2817.416666666668, + "max": 23150.666666666668, + "mean": 7301.620833333333, + "median": 5446.866666666667, + "min": 3190.3333333333335, + "non_zero_mean": 7301.620833333333, + "non_zero_median": 5446.866666666667, + "percentile25": 4973.166666666666, + "percentile75": 7790.583333333334, + "percentile90": 11482.626666666667, + "percentile99": 23119.082666666665, + "percentile999": 23147.508266666668, + "range": 19960.333333333336, + "samples": 64, + "stdev": 4288.513041922659, + "sum": 467303.73333333334 + }, + "pg_stat_database_tup_inserted": { + "iqr": 354.9833333333334, + "max": 793.8666666666667, + "mean": 531.9572916666667, + "median": 692.9000000000001, + "min": 0.06666666666666667, + "non_zero_mean": 531.9572916666667, + "non_zero_median": 692.9000000000001, + "percentile25": 362.93333333333334, + "percentile75": 717.9166666666667, + "percentile90": 747.34, + "percentile99": 784.4166666666666, + "percentile999": 792.9216666666667, + "range": 793.8, + "samples": 64, + "stdev": 259.2620065857364, + "sum": 34045.266666666656 + }, + "pg_stat_database_tup_returned": { + "iqr": 3083.2, + "max": 29671.0, + "mean": 9149.802083333334, + "median": 6740.6, + "min": 4255.666666666667, + "non_zero_mean": 9149.802083333334, + "non_zero_median": 6740.6, + "percentile25": 6283.683333333333, + "percentile75": 9366.883333333333, + "percentile90": 15923.753333333336, + "percentile99": 28693.155999999995, + "percentile999": 29573.215600000007, + "range": 25415.333333333332, + "samples": 64, + "stdev": 5450.968987799948, + "sum": 585587.3333333334 + }, + "pg_stat_database_tup_updated": { + "iqr": 10.183333333333334, + "max": 122.13333333333334, + "mean": 10.182291666666666, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 11.636904761904763, + "non_zero_median": 1.4, + "percentile25": 0.18333333333333335, + "percentile75": 10.366666666666667, + "percentile90": 27.18, + "percentile99": 114.3213333333333, + "percentile999": 121.3521333333334, + "range": 122.13333333333334, + "samples": 64, + "stdev": 24.840526261121223, + "sum": 651.6666666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 14.883333333333333, + "max": 148.4, + "mean": 59.9375, + "median": 49.8, + "min": 13.666666666666666, + "non_zero_mean": 59.9375, + "non_zero_median": 49.8, + "percentile25": 47.21666666666667, + "percentile75": 62.1, + "percentile90": 119.76666666666668, + "percentile99": 138.31999999999996, + "percentile999": 147.39200000000008, + "range": 134.73333333333335, + "samples": 64, + "stdev": 28.77951425783812, + "sum": 3836.0000000000014 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 4.266666666666671 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.33333333333333337, + "mean": 0.009375000000000001, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.30000000000000004, + "non_zero_median": 0.30000000000000004, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.29133333333333317, + "percentile999": 0.32913333333333367, + "range": 0.33333333333333337, + "samples": 64, + "stdev": 0.05294459851074371, + "sum": 0.6000000000000001 + } + }, + "writes_completed": { + "total": { + "iqr": 18.58333333333333, + "max": 127.13333333333334, + "mean": 79.02395833333334, + "median": 81.03333333333333, + "min": 8.533333333333333, + "non_zero_mean": 79.02395833333334, + "non_zero_median": 81.03333333333333, + "percentile25": 71.26666666666667, + "percentile75": 89.85, + "percentile90": 110.08000000000001, + "percentile99": 123.43733333333331, + "percentile999": 126.76373333333336, + "range": 118.60000000000001, + "samples": 64, + "stdev": 24.978069724293658, + "sum": 5057.533333333335 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,10,8,11,9,14,18,20,21,24,19,25,22,23,26,27,28,29,31,34,32,35,30,33,36,37,39,38,40,43,42,41,44,45,46,47,48,52,49,51,50,53,54,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-10T09:16:43.449971Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T09:16:44.122743Z 2021-12-10T09:16:44.115853Z 2021-12-10T09:16:44.095276Z 2021-12-10T09:16:43.530179Z 2021-12-10T09:16:43.463665Z 2021-12-10T09:16:43.450472Z 2021-12-10T09:16:43.449971Z 2021-12-10T09:16:45.470693Z 2021-12-10T09:16:44.194056Z 2021-12-10T09:16:44.126571Z 2021-12-10T09:21:02.379603Z 2021-12-10T09:21:02.285372Z 2021-12-10T09:21:01.476576Z 2021-12-10T09:21:01.448746Z 2021-12-10T09:21:01.424607Z 2021-12-10T09:21:01.395778Z 2021-12-10T09:21:01.346291Z 2021-12-10T09:21:01.339374Z 2021-12-10T09:21:01.336494Z 2021-12-10T09:21:01.249995Z 2021-12-10T09:23:56.567042Z 2021-12-10T09:23:56.431169Z 2021-12-10T09:23:56.019734Z 2021-12-10T09:23:55.978518Z 2021-12-10T09:23:55.977115Z 2021-12-10T09:23:55.975525Z 2021-12-10T09:23:55.962062Z 2021-12-10T09:23:55.948940Z 2021-12-10T09:23:55.946559Z 2021-12-10T09:23:55.844945Z 2021-12-10T09:27:00.714862Z 2021-12-10T09:27:00.706235Z 2021-12-10T09:27:00.293754Z 2021-12-10T09:27:00.274846Z 2021-12-10T09:27:00.258584Z 2021-12-10T09:27:00.256700Z 2021-12-10T09:27:00.251904Z 2021-12-10T09:27:00.251030Z 2021-12-10T09:27:00.135431Z 2021-12-10T09:27:00.133969Z 2021-12-10T09:30:05.393964Z 2021-12-10T09:30:04.944934Z 2021-12-10T09:30:04.599456Z 2021-12-10T09:30:04.548643Z 2021-12-10T09:30:04.540787Z 2021-12-10T09:30:04.453854Z 2021-12-10T09:30:04.451941Z 2021-12-10T09:30:04.451835Z 2021-12-10T09:30:04.449967Z 2021-12-10T09:30:04.332053Z", + "created_diff": 1.1030462, + "ended": "2021-12-10T09:32:41.850664Z", + "error_job_ids": " []", + "event_first": "2021-12-10T09:16:54.457631Z 2021-12-10T09:18:11.483767Z 2021-12-10T09:16:56.477839Z 2021-12-10T09:18:25.576649Z 2021-12-10T09:16:49.632113Z 2021-12-10T09:17:33.951295Z 2021-12-10T09:16:50.146452Z 2021-12-10T09:17:44.529853Z 2021-12-10T09:16:58.752688Z 2021-12-10T09:17:56.542000Z 2021-12-10T09:21:16.345384Z 2021-12-10T09:21:16.446127Z 2021-12-10T09:21:08.647641Z 2021-12-10T09:21:11.993977Z 2021-12-10T09:21:10.925488Z 2021-12-10T09:21:10.247981Z 2021-12-10T09:21:08.450811Z 2021-12-10T09:21:08.474548Z 2021-12-10T09:21:08.913946Z 2021-12-10T09:21:09.478522Z 2021-12-10T09:24:23.439780Z 2021-12-10T09:24:25.197589Z 2021-12-10T09:24:02.430320Z 2021-12-10T09:24:06.242587Z 2021-12-10T09:24:02.684717Z 2021-12-10T09:24:02.909731Z 2021-12-10T09:24:01.914842Z 2021-12-10T09:24:02.152677Z 2021-12-10T09:24:02.430320Z 2021-12-10T09:24:01.909580Z 2021-12-10T09:27:26.586667Z 2021-12-10T09:27:29.291596Z 2021-12-10T09:27:08.119946Z 2021-12-10T09:27:08.981406Z 2021-12-10T09:27:08.374463Z 2021-12-10T09:27:08.489673Z 2021-12-10T09:27:06.605475Z 2021-12-10T09:27:06.558271Z 2021-12-10T09:27:06.172253Z 2021-12-10T09:27:06.058784Z 2021-12-10T09:30:17.684768Z 2021-12-10T09:30:22.952724Z 2021-12-10T09:30:16.635350Z 2021-12-10T09:30:14.488570Z 2021-12-10T09:30:11.770111Z 2021-12-10T09:30:12.741902Z 2021-12-10T09:30:12.531685Z 2021-12-10T09:30:12.702756Z 2021-12-10T09:30:10.511268Z 2021-12-10T09:30:11.465634Z", + "event_last": "2021-12-10T09:19:28.354939Z 2021-12-10T09:20:27.652159Z 2021-12-10T09:19:30.814695Z 2021-12-10T09:20:38.405108Z 2021-12-10T09:19:22.276457Z 2021-12-10T09:20:03.953518Z 2021-12-10T09:19:23.907262Z 2021-12-10T09:20:11.196687Z 2021-12-10T09:19:31.585325Z 2021-12-10T09:20:20.042497Z 2021-12-10T09:23:40.240455Z 2021-12-10T09:23:38.675442Z 2021-12-10T09:23:31.997763Z 2021-12-10T09:23:36.083705Z 2021-12-10T09:23:33.824738Z 2021-12-10T09:23:35.782622Z 2021-12-10T09:23:33.160720Z 2021-12-10T09:23:34.184228Z 2021-12-10T09:23:32.380412Z 2021-12-10T09:23:34.837403Z 2021-12-10T09:26:40.535207Z 2021-12-10T09:26:42.310504Z 2021-12-10T09:26:25.989261Z 2021-12-10T09:26:29.505847Z 2021-12-10T09:26:25.518753Z 2021-12-10T09:26:27.052412Z 2021-12-10T09:26:23.743060Z 2021-12-10T09:26:26.325742Z 2021-12-10T09:26:27.413439Z 2021-12-10T09:26:26.879591Z 2021-12-10T09:29:44.473515Z 2021-12-10T09:29:46.152517Z 2021-12-10T09:29:32.731462Z 2021-12-10T09:29:34.726768Z 2021-12-10T09:29:31.402561Z 2021-12-10T09:29:34.875372Z 2021-12-10T09:29:30.728313Z 2021-12-10T09:29:33.009968Z 2021-12-10T09:29:30.605294Z 2021-12-10T09:29:33.440684Z 2021-12-10T09:32:39.714675Z 2021-12-10T09:32:41.850664Z 2021-12-10T09:32:38.001151Z 2021-12-10T09:32:37.905581Z 2021-12-10T09:32:33.814345Z 2021-12-10T09:32:35.906940Z 2021-12-10T09:32:34.060245Z 2021-12-10T09:32:36.233232Z 2021-12-10T09:32:32.523381Z 2021-12-10T09:32:35.302356Z", + "failed_job_ids": " []", + "finished": "2021-12-10T09:19:31.883043Z 2021-12-10T09:20:31.122258Z 2021-12-10T09:19:35.138730Z 2021-12-10T09:20:39.797889Z 2021-12-10T09:19:26.285319Z 2021-12-10T09:20:08.066035Z 2021-12-10T09:19:28.079477Z 2021-12-10T09:20:15.187352Z 2021-12-10T09:19:35.535047Z 2021-12-10T09:20:23.691376Z 2021-12-10T09:23:42.115582Z 2021-12-10T09:23:42.411414Z 2021-12-10T09:23:36.134754Z 2021-12-10T09:23:40.248359Z 2021-12-10T09:23:37.947787Z 2021-12-10T09:23:39.807258Z 2021-12-10T09:23:37.707796Z 2021-12-10T09:23:37.580833Z 2021-12-10T09:23:36.037967Z 2021-12-10T09:23:38.405573Z 2021-12-10T09:26:41.857380Z 2021-12-10T09:26:45.362462Z 2021-12-10T09:26:30.088812Z 2021-12-10T09:26:33.540761Z 2021-12-10T09:26:29.033752Z 2021-12-10T09:26:30.984819Z 2021-12-10T09:26:27.553909Z 2021-12-10T09:26:29.723933Z 2021-12-10T09:26:31.830010Z 2021-12-10T09:26:30.798268Z 2021-12-10T09:29:48.961720Z 2021-12-10T09:29:50.032838Z 2021-12-10T09:29:36.855758Z 2021-12-10T09:29:38.760227Z 2021-12-10T09:29:35.402551Z 2021-12-10T09:29:38.603914Z 2021-12-10T09:29:34.363612Z 2021-12-10T09:29:37.080407Z 2021-12-10T09:29:34.975624Z 2021-12-10T09:29:36.885962Z 2021-12-10T09:32:42.322209Z 2021-12-10T09:32:45.745111Z 2021-12-10T09:32:41.891928Z 2021-12-10T09:32:42.093252Z 2021-12-10T09:32:37.518228Z 2021-12-10T09:32:39.832073Z 2021-12-10T09:32:38.266589Z 2021-12-10T09:32:40.453682Z 2021-12-10T09:32:35.917987Z 2021-12-10T09:32:39.420046Z", + "finished_diff": 24.638184, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 235.133163, + "mean": 161.75313016, + "min": 145.952724 + }, + "jobs_events_duration": { + "max": 154.336856, + "mean": 144.01157636000005, + "min": 132.828459 + }, + "jobs_events_lag": { + "max": -1.322173, + "mean": -3.7450539599999995, + "min": -4.547076 + }, + "jobs_waiting": { + "max": 19.348667, + "mean": 2.71772984, + "min": 0.870303 + }, + "modified": "2021-12-10T09:16:45.393476Z 2021-12-10T09:16:45.330630Z 2021-12-10T09:16:45.285094Z 2021-12-10T09:16:44.123937Z 2021-12-10T09:16:44.090615Z 2021-12-10T09:16:44.051294Z 2021-12-10T09:16:44.019341Z 2021-12-10T09:16:46.367436Z 2021-12-10T09:16:45.459158Z 2021-12-10T09:16:45.423743Z 2021-12-10T09:21:03.435411Z 2021-12-10T09:21:03.389331Z 2021-12-10T09:21:02.092375Z 2021-12-10T09:21:02.059906Z 2021-12-10T09:21:02.027141Z 2021-12-10T09:21:01.994694Z 2021-12-10T09:21:01.961405Z 2021-12-10T09:21:01.920852Z 2021-12-10T09:21:01.884585Z 2021-12-10T09:21:01.847177Z 2021-12-10T09:24:15.500842Z 2021-12-10T09:24:15.453853Z 2021-12-10T09:23:56.639445Z 2021-12-10T09:23:56.608329Z 2021-12-10T09:23:56.577002Z 2021-12-10T09:23:56.543249Z 2021-12-10T09:23:56.508108Z 2021-12-10T09:23:56.477770Z 2021-12-10T09:23:56.445250Z 2021-12-10T09:23:56.406937Z 2021-12-10T09:27:17.468496Z 2021-12-10T09:27:17.323672Z 2021-12-10T09:27:00.916501Z 2021-12-10T09:27:00.885546Z 2021-12-10T09:27:00.850154Z 2021-12-10T09:27:00.809742Z 2021-12-10T09:27:00.772103Z 2021-12-10T09:27:00.741827Z 2021-12-10T09:27:00.711174Z 2021-12-10T09:27:00.678425Z 2021-12-10T09:30:06.628776Z 2021-12-10T09:30:06.527537Z 2021-12-10T09:30:06.481663Z 2021-12-10T09:30:05.136268Z 2021-12-10T09:30:05.105186Z 2021-12-10T09:30:05.073465Z 2021-12-10T09:30:05.039009Z 2021-12-10T09:30:05.005030Z 2021-12-10T09:30:04.974940Z 2021-12-10T09:30:04.940720Z", + "modified_diff": 8.301672200000002, + "started": "2021-12-10T09:16:45.809707Z 2021-12-10T09:16:45.757109Z 2021-12-10T09:16:45.665053Z 2021-12-10T09:16:44.664726Z 2021-12-10T09:16:44.487262Z 2021-12-10T09:16:44.612114Z 2021-12-10T09:16:44.320274Z 2021-12-10T09:16:46.553227Z 2021-12-10T09:16:46.121801Z 2021-12-10T09:16:45.969411Z 2021-12-10T09:21:04.151664Z 2021-12-10T09:21:04.013688Z 2021-12-10T09:21:03.148270Z 2021-12-10T09:21:03.013991Z 2021-12-10T09:21:02.821810Z 2021-12-10T09:21:02.761773Z 2021-12-10T09:21:02.664864Z 2021-12-10T09:21:02.575449Z 2021-12-10T09:21:02.444318Z 2021-12-10T09:21:02.360377Z 2021-12-10T09:24:15.904656Z 2021-12-10T09:24:15.779836Z 2021-12-10T09:23:57.422907Z 2021-12-10T09:23:57.383123Z 2021-12-10T09:23:57.243864Z 2021-12-10T09:23:57.135142Z 2021-12-10T09:23:57.042174Z 2021-12-10T09:23:56.963919Z 2021-12-10T09:23:56.902500Z 2021-12-10T09:23:56.813891Z 2021-12-10T09:27:18.587960Z 2021-12-10T09:27:18.222766Z 2021-12-10T09:27:01.819114Z 2021-12-10T09:27:01.713372Z 2021-12-10T09:27:01.602754Z 2021-12-10T09:27:01.514863Z 2021-12-10T09:27:01.414488Z 2021-12-10T09:27:01.334733Z 2021-12-10T09:27:01.251623Z 2021-12-10T09:27:01.170119Z 2021-12-10T09:30:07.323274Z 2021-12-10T09:30:07.303315Z 2021-12-10T09:30:07.059095Z 2021-12-10T09:30:05.836618Z 2021-12-10T09:30:05.719885Z 2021-12-10T09:30:05.621933Z 2021-12-10T09:30:05.542807Z 2021-12-10T09:30:05.457975Z 2021-12-10T09:30:05.374778Z 2021-12-10T09:30:05.304793Z", + "started_diff": 8.510265400000002, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-10T09:35:20.983422+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-launching.json new file mode 100644 index 00000000..94c87746 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T08_38_02_566_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-10T10:22:49.092845+00:00", + "golden": "true", + "id": "run-2021-12-10T08_38_02_566_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17830.666666666668, + "max": 31218.0, + "mean": 9805.109956709957, + "median": 4385.133333333333, + "min": 1361.8, + "non_zero_mean": 9805.109956709957, + "non_zero_median": 4385.133333333333, + "percentile25": 1780.6, + "percentile75": 19611.266666666666, + "percentile90": 22431.54666666667, + "percentile99": 29484.034666666656, + "percentile999": 31044.603466666682, + "range": 29856.2, + "samples": 77, + "stdev": 9379.882844988186, + "sum": 754993.4666666668 + } + }, + "cpu": { + "system": { + "iqr": 1.373333333333333, + "max": 1.7713333333333328, + "mean": 0.6630129870129869, + "median": 0.23066666666666719, + "min": 0.02799999999999917, + "non_zero_mean": 0.6630129870129869, + "non_zero_median": 0.23066666666666719, + "percentile25": 0.03733333333333349, + "percentile75": 1.4106666666666665, + "percentile90": 1.6381333333333326, + "percentile99": 1.7606933333333354, + "percentile999": 1.770269333333333, + "range": 1.7433333333333336, + "samples": 77, + "stdev": 0.6938726108742987, + "sum": 51.05199999999998 + }, + "user": { + "iqr": 5.751333333333318, + "max": 6.475333333333295, + "mean": 2.811142857142857, + "median": 0.8766666666666576, + "min": 0.11399999999998726, + "non_zero_mean": 2.811142857142857, + "non_zero_median": 0.8766666666666576, + "percentile25": 0.31400000000001, + "percentile75": 6.065333333333328, + "percentile90": 6.369333333333328, + "percentile99": 6.431253333333324, + "percentile999": 6.470925333333298, + "range": 6.361333333333308, + "samples": 77, + "stdev": 2.747280907423569, + "sum": 216.45799999999994 + } + }, + "entropy_available_bits": { + "iqr": 3.8666666666666667, + "max": 214.66666666666666, + "mean": 23.606060606060606, + "median": 0.8666666666666667, + "min": 0.0, + "non_zero_mean": 25.24537037037037, + "non_zero_median": 0.9666666666666667, + "percentile25": 0.6666666666666666, + "percentile75": 4.533333333333333, + "percentile90": 86.1466666666678, + "percentile99": 214.16, + "percentile999": 214.61599999999999, + "range": 214.66666666666666, + "samples": 77, + "stdev": 64.11111259452132, + "sum": 1817.6666666666667 + }, + "memory": { + "used": { + "iqr": 5371330560.0, + "max": 17858019328.0, + "mean": 8756732967.896105, + "median": 6533500928.0, + "min": 5233876992.0, + "non_zero_mean": 8756732967.896105, + "non_zero_median": 6533500928.0, + "percentile25": 6121668608.0, + "percentile75": 11492999168.0, + "percentile90": 15298400256.000002, + "percentile99": 16894670274.559994, + "percentile999": 17761684422.65601, + "range": 12624142336.0, + "samples": 77, + "stdev": 3826723734.2964654, + "sum": 674268438528.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.0666666666666667, + "mean": 0.017316017316017316, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6666666666666666, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.45866666666666256, + "percentile999": 1.005866666666672, + "range": 1.0666666666666667, + "samples": 77, + "stdev": 0.12491053987179793, + "sum": 1.3333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 871.3333333333334, + "max": 1883.8666666666666, + "mean": 461.72900432900434, + "median": 83.19999999999999, + "min": 0.6000000000000001, + "non_zero_mean": 461.72900432900434, + "non_zero_median": 83.19999999999999, + "percentile25": 3.533333333333333, + "percentile75": 874.8666666666667, + "percentile90": 1418.9066666666668, + "percentile99": 1796.1119999999994, + "percentile999": 1875.0912000000008, + "range": 1883.2666666666667, + "samples": 77, + "stdev": 600.6025765930854, + "sum": 35553.13333333334 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2250.8, + "max": 3541.4, + "mean": 1709.5047619047618, + "median": 953.8666666666667, + "min": 336.06666666666666, + "non_zero_mean": 1709.5047619047618, + "non_zero_median": 953.8666666666667, + "percentile25": 645.0666666666667, + "percentile75": 2895.866666666667, + "percentile90": 3167.0666666666666, + "percentile99": 3466.5146666666665, + "percentile999": 3533.9114666666674, + "range": 3205.3333333333335, + "samples": 77, + "stdev": 1124.576063823599, + "sum": 131631.8666666667 + } + }, + "cpu": { + "system": { + "iqr": 0.21799999999999972, + "max": 0.3573333333333333, + "mean": 0.13152380952380952, + "median": 0.04599999999999985, + "min": 0.009333333333333372, + "non_zero_mean": 0.13152380952380952, + "non_zero_median": 0.04599999999999985, + "percentile25": 0.02466666666666697, + "percentile75": 0.2426666666666667, + "percentile90": 0.3079999999999998, + "percentile99": 0.35733333333333256, + "percentile999": 0.3573333333333332, + "range": 0.3479999999999999, + "samples": 77, + "stdev": 0.12046493057627551, + "sum": 10.127333333333333 + }, + "user": { + "iqr": 0.2826666666666673, + "max": 0.7693333333333319, + "mean": 0.19842424242424242, + "median": 0.05666666666666628, + "min": 0.009333333333335267, + "non_zero_mean": 0.19842424242424242, + "non_zero_median": 0.05666666666666628, + "percentile25": 0.0386666666666675, + "percentile75": 0.3213333333333348, + "percentile90": 0.3961333333333331, + "percentile99": 0.7358933333333317, + "percentile999": 0.7659893333333322, + "range": 0.7599999999999966, + "samples": 77, + "stdev": 0.19790750125676088, + "sum": 15.278666666666668 + } + }, + "entropy_available_bits": { + "iqr": 2.8, + "max": 213.2, + "mean": 11.909090909090908, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 15.810344827586206, + "non_zero_median": 0.5333333333333333, + "percentile25": 0.2, + "percentile75": 3.0, + "percentile90": 4.573333333333335, + "percentile99": 211.832, + "percentile999": 213.0632, + "range": 213.2, + "samples": 77, + "stdev": 46.02717721084942, + "sum": 917.0 + }, + "memory": { + "used": { + "iqr": 488894464.0, + "max": 1161281536.0, + "mean": 583367214.5454545, + "median": 406761472.0, + "min": 366067712.0, + "non_zero_mean": 583367214.5454545, + "non_zero_median": 406761472.0, + "percentile25": 375857152.0, + "percentile75": 864751616.0, + "percentile90": 1017463603.2, + "percentile99": 1156163829.76, + "percentile999": 1160769765.376, + "range": 795213824.0, + "samples": 77, + "stdev": 266240058.96656424, + "sum": 44919275520.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 2184.5333333333333, + "max": 56118535.4, + "mean": 745877.0415584416, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2734882.4857142856, + "non_zero_median": 66082.13333333333, + "percentile25": 0.0, + "percentile75": 2184.5333333333333, + "percentile90": 74929.4933333334, + "percentile99": 13600853.061333047, + "percentile999": 51866767.1661337, + "range": 56118535.4, + "samples": 77, + "stdev": 6393437.05146414, + "sum": 57432532.199999996 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 1.347001, + "mean": 0.04941105194805195, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.3804651, + "non_zero_median": 0.104445, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.015356000000000048, + "percentile99": 1.2722109199999996, + "percentile999": 1.3395219920000005, + "range": 1.347001, + "samples": 77, + "stdev": 0.22051265016105062, + "sum": 3.804651 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.653267, + "max": 7.439732, + "mean": 0.6089116363636363, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.4207938181818183, + "non_zero_median": 0.920959, + "percentile25": 0.0, + "percentile75": 0.653267, + "percentile90": 1.3908906, + "percentile99": 6.860589199999996, + "percentile999": 7.381817720000005, + "range": 7.439732, + "samples": 77, + "stdev": 1.4028084473030193, + "sum": 46.88619599999999 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.2, + "mean": 0.12380952380952381, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.3833333333333333, + "non_zero_median": 2.4333333333333336, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.743999999999997, + "percentile999": 3.1544000000000043, + "range": 3.2, + "samples": 77, + "stdev": 0.5514164944721366, + "sum": 9.533333333333335 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9998.533333333333, + "mean": 443.0112554112554, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8527.966666666667, + "non_zero_median": 8691.266666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9961.597333333333, + "percentile999": 9994.839733333332, + "range": 9998.533333333333, + "samples": 77, + "stdev": 1934.4763764495133, + "sum": 34111.86666666667 + }, + "pg_stat_database_blks_hit": { + "iqr": 24297.466666666667, + "max": 98726.33333333333, + "mean": 21852.587012987013, + "median": 10172.4, + "min": 1194.3333333333333, + "non_zero_mean": 21852.587012987013, + "non_zero_median": 10172.4, + "percentile25": 7268.2, + "percentile75": 31565.666666666668, + "percentile90": 45045.45333333334, + "percentile99": 89692.1626666666, + "percentile999": 97822.91626666674, + "range": 97532.0, + "samples": 77, + "stdev": 21748.456921335495, + "sum": 1682649.199999999 + }, + "pg_stat_database_blks_read": { + "iqr": 3.1333333333333333, + "max": 13.466666666666667, + "mean": 2.0761904761904764, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 3.401418439716312, + "non_zero_median": 2.1333333333333333, + "percentile25": 0.0, + "percentile75": 3.1333333333333333, + "percentile90": 6.4266666666666845, + "percentile99": 11.642666666666655, + "percentile999": 13.284266666666683, + "range": 13.466666666666667, + "samples": 77, + "stdev": 3.144785236779081, + "sum": 159.8666666666667 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 77, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.1333333333333333, + "max": 13.466666666666667, + "mean": 2.0761904761904764, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 3.401418439716312, + "non_zero_median": 2.1333333333333333, + "percentile25": 0.0, + "percentile75": 3.1333333333333333, + "percentile90": 6.4266666666666845, + "percentile99": 11.642666666666655, + "percentile999": 13.284266666666683, + "range": 13.466666666666667, + "samples": 77, + "stdev": 3.144785236779081, + "sum": 159.8666666666667 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4666666666666666, + "mean": 0.15670995670995672, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.9282051282051282, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9333333333333333, + "percentile99": 1.3653333333333326, + "percentile999": 1.4565333333333341, + "range": 1.4666666666666666, + "samples": 77, + "stdev": 0.36972666271024834, + "sum": 12.066666666666666 + }, + "pg_stat_database_tup_fetched": { + "iqr": 13062.266666666665, + "max": 49501.26666666667, + "mean": 11590.277922077923, + "median": 5174.266666666666, + "min": 583.6666666666666, + "non_zero_mean": 11590.277922077923, + "non_zero_median": 5174.266666666666, + "percentile25": 3765.3333333333335, + "percentile75": 16827.6, + "percentile90": 23900.50666666667, + "percentile99": 45683.33066666664, + "percentile999": 49119.4730666667, + "range": 48917.600000000006, + "samples": 77, + "stdev": 11239.080272574816, + "sum": 892451.3999999999 + }, + "pg_stat_database_tup_inserted": { + "iqr": 28.666666666666668, + "max": 99.93333333333334, + "mean": 16.205194805194804, + "median": 5.4, + "min": 0.0, + "non_zero_mean": 26.548936170212766, + "non_zero_median": 18.733333333333334, + "percentile25": 0.0, + "percentile75": 28.666666666666668, + "percentile90": 46.040000000000134, + "percentile99": 79.7173333333332, + "percentile999": 97.91173333333352, + "range": 99.93333333333334, + "samples": 77, + "stdev": 23.14156728490121, + "sum": 1247.8000000000002 + }, + "pg_stat_database_tup_returned": { + "iqr": 18748.8, + "max": 74822.46666666666, + "mean": 18513.95670995671, + "median": 9536.2, + "min": 1350.7333333333333, + "non_zero_mean": 18513.95670995671, + "non_zero_median": 9536.2, + "percentile25": 6348.2, + "percentile75": 25097.0, + "percentile90": 38059.38666666668, + "percentile99": 73632.61066666666, + "percentile999": 74703.48106666667, + "range": 73471.73333333332, + "samples": 77, + "stdev": 17582.900177911644, + "sum": 1425574.6666666665 + }, + "pg_stat_database_tup_updated": { + "iqr": 30.46666666666667, + "max": 65.53333333333333, + "mean": 16.683982683982684, + "median": 7.666666666666667, + "min": 0.0, + "non_zero_mean": 18.892156862745097, + "non_zero_median": 12.666666666666666, + "percentile25": 0.26666666666666666, + "percentile75": 30.733333333333334, + "percentile90": 40.480000000000004, + "percentile99": 62.03733333333331, + "percentile999": 65.18373333333336, + "range": 65.53333333333333, + "samples": 77, + "stdev": 18.541038416629952, + "sum": 1284.666666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 263.26666666666665, + "max": 580.2666666666667, + "mean": 153.84588744588746, + "median": 48.333333333333336, + "min": 2.466666666666667, + "non_zero_mean": 153.84588744588746, + "non_zero_median": 48.333333333333336, + "percentile25": 9.6, + "percentile75": 272.8666666666667, + "percentile90": 352.8266666666667, + "percentile99": 547.6879999999998, + "percentile999": 577.0088000000003, + "range": 577.8, + "samples": 77, + "stdev": 164.83070963081036, + "sum": 11846.133333333331 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.13333333333333333, + "mean": 0.06753246753246753, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06753246753246753, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.08266666666666633, + "percentile999": 0.1282666666666671, + "range": 0.06666666666666667, + "samples": 77, + "stdev": 0.007597371763975863, + "sum": 5.200000000000001 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003463203463203463, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06399999999999864, + "percentile999": 0.24640000000000176, + "range": 0.26666666666666666, + "samples": 77, + "stdev": 0.030389487055903455, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 90.93333333333332, + "max": 151.86666666666667, + "mean": 57.829437229437225, + "median": 36.06666666666666, + "min": 1.3333333333333333, + "non_zero_mean": 57.829437229437225, + "non_zero_median": 36.06666666666666, + "percentile25": 12.2, + "percentile75": 103.13333333333333, + "percentile90": 130.25333333333333, + "percentile99": 148.31999999999996, + "percentile999": 151.51200000000003, + "range": 150.53333333333333, + "samples": 77, + "stdev": 49.65574882179697, + "sum": 4452.866666666666 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "62,61,60,69,78,70,76,63,64,75,65,73,67,72,66,77,71,79,81,80,82,85,83,84,86,88,90,92,95,91,94,96,99,97,102,101,104,100,103,107,109,108,111,112,98,105,116,114,117,106,118,124,127,120,126,130,131,123,133,125,121,135,129,138,137,134,139,141,142,143,148,145,150,153,136,149,144,152,140,147,154,155,158,157,146,159,151,161,162,160,164,156,165,163,166,168,167,170,169,171,178,172,177,175,176,179,184,190,181,180,185,187,192,174,186,183,182,188,189,198,173,193,196,191,195,197,199,194,200,201,204,205,207,209,202,203,208,213,210,214,211,206,218,215,217,212,220,221,223,222,219,224,225,226,228,230,229,227,242,237,241,216,234,232,244,231,233,243,246,245,240,238,236,247,249,250,251,252,255,248,235,239,256,254,253,258,261,263,259,262,257,267,266,264,265,260,268,269,270,271,272,276,278,284,282,275,273,274,287,280,286,288,299,283,279,281,290,294,289,277,285,291,298,297,292,300,303,293,302,304,301,295,305,296,310,308,307,306,312,313,311,314,315,316,319,318,309,317,320,323,321,324,327,322,328,326,325,329,336,332,334,330,338,337,335,333,331,340,339,343,347,342,341,345,344,349,346,354,350,352,351,357,355,361,358,353,348,359,356,363,364,366,360,365,362,368,367,369,370,371,373,376,377,375,379,378,372,381,383,374,390,380,382,385,387,388,386,396,389,401,400,391,398,384,395,399,392,393,407,397,409,402,394,406,404,403,408,410,405,413,412,411,415,414,416,419,417,418,422,420,425,421,423,427,426,431,430,428,424,429,432,437,433,436,438,434,440,435,442,446,444,439,441,443,445,450,449,455,447,448,453,456,454,452,458,451,457,459,462,461,466,467,463,468,465,460,464,469,470,471,474,473,480,472,476,477,475,481,479,486,482,483,495,478,489,487,485,490,500,499,497,488,484,494,501,496,493,492,491,498,502,504,505,503,509,507,508,513,514,510,506,518,511,520,512,524,517,519,516,523,521,515,522,526,528,527,525,531,529,537,536,535,530,538,534,533,539,547,541,545,544,553,549,552,556,554,546,558,540,559,532,560,561,551,550,555,543,562,565,542,563,566,564,567,548,569,570,571,557,568", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-10T09:38:58.034336Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T09:39:03.617462Z 2021-12-10T09:39:03.300734Z 2021-12-10T09:39:03.221291Z 2021-12-10T09:39:03.207552Z 2021-12-10T09:39:03.106522Z 2021-12-10T09:39:03.076721Z 2021-12-10T09:39:02.713822Z 2021-12-10T09:39:02.707252Z 2021-12-10T09:39:02.303633Z 2021-12-10T09:39:02.174304Z 2021-12-10T09:39:01.893555Z 2021-12-10T09:39:01.729853Z 2021-12-10T09:39:01.706761Z 2021-12-10T09:39:01.572839Z 2021-12-10T09:39:01.515025Z 2021-12-10T09:39:01.414301Z 2021-12-10T09:39:01.183364Z 2021-12-10T09:39:00.830867Z 2021-12-10T09:39:00.812701Z 2021-12-10T09:39:00.705834Z 2021-12-10T09:39:00.640942Z 2021-12-10T09:39:00.527072Z 2021-12-10T09:39:00.394164Z 2021-12-10T09:39:00.357167Z 2021-12-10T09:38:59.983490Z 2021-12-10T09:38:59.934411Z 2021-12-10T09:38:59.875410Z 2021-12-10T09:38:59.740983Z 2021-12-10T09:38:59.693626Z 2021-12-10T09:38:59.642213Z 2021-12-10T09:38:59.577274Z 2021-12-10T09:38:59.500605Z 2021-12-10T09:38:59.248964Z 2021-12-10T09:38:58.904935Z 2021-12-10T09:38:58.034336Z 2021-12-10T09:39:28.833870Z 2021-12-10T09:39:28.766786Z 2021-12-10T09:39:27.878896Z 2021-12-10T09:39:24.822919Z 2021-12-10T09:39:23.031884Z 2021-12-10T09:39:21.911984Z 2021-12-10T09:39:20.205837Z 2021-12-10T09:39:18.352770Z 2021-12-10T09:39:16.422325Z 2021-12-10T09:39:16.363674Z 2021-12-10T09:39:16.290542Z 2021-12-10T09:39:15.930394Z 2021-12-10T09:39:15.657297Z 2021-12-10T09:39:13.991181Z 2021-12-10T09:39:13.648682Z 2021-12-10T09:39:13.329542Z 2021-12-10T09:39:12.383013Z 2021-12-10T09:39:11.237871Z 2021-12-10T09:39:10.805674Z 2021-12-10T09:39:10.718099Z 2021-12-10T09:39:10.701944Z 2021-12-10T09:39:10.621761Z 2021-12-10T09:39:10.543218Z 2021-12-10T09:39:10.530774Z 2021-12-10T09:39:10.457298Z 2021-12-10T09:39:10.449872Z 2021-12-10T09:39:10.421773Z 2021-12-10T09:39:10.248123Z 2021-12-10T09:39:10.119034Z 2021-12-10T09:39:09.985293Z 2021-12-10T09:39:09.635012Z 2021-12-10T09:39:09.499988Z 2021-12-10T09:39:09.477900Z 2021-12-10T09:39:09.416750Z 2021-12-10T09:39:09.215763Z 2021-12-10T09:39:09.006401Z 2021-12-10T09:39:08.808803Z 2021-12-10T09:39:08.449366Z 2021-12-10T09:39:08.320544Z 2021-12-10T09:39:08.219377Z 2021-12-10T09:39:08.013374Z 2021-12-10T09:39:07.802885Z 2021-12-10T09:39:07.727993Z 2021-12-10T09:39:07.596511Z 2021-12-10T09:39:07.442210Z 2021-12-10T09:39:07.287990Z 2021-12-10T09:39:07.178955Z 2021-12-10T09:39:06.743771Z 2021-12-10T09:39:06.557494Z 2021-12-10T09:39:06.279840Z 2021-12-10T09:39:06.242911Z 2021-12-10T09:39:06.213832Z 2021-12-10T09:39:06.186003Z 2021-12-10T09:39:05.314262Z 2021-12-10T09:39:05.137282Z 2021-12-10T09:39:05.001227Z 2021-12-10T09:39:04.973212Z 2021-12-10T09:39:04.641192Z 2021-12-10T09:39:04.436436Z 2021-12-10T09:39:04.274297Z 2021-12-10T09:39:04.045542Z 2021-12-10T09:39:04.012182Z 2021-12-10T09:39:03.998950Z 2021-12-10T09:39:03.996581Z 2021-12-10T09:39:03.633098Z 2021-12-10T09:44:12.283677Z 2021-12-10T09:44:11.881571Z 2021-12-10T09:44:10.398990Z 2021-12-10T09:44:09.181327Z 2021-12-10T09:44:08.654481Z 2021-12-10T09:44:08.652310Z 2021-12-10T09:44:08.316841Z 2021-12-10T09:44:08.135542Z 2021-12-10T09:44:07.881196Z 2021-12-10T09:44:07.586955Z 2021-12-10T09:44:07.388726Z 2021-12-10T09:44:07.294864Z 2021-12-10T09:44:07.271527Z 2021-12-10T09:44:07.131205Z 2021-12-10T09:44:06.984738Z 2021-12-10T09:44:06.946806Z 2021-12-10T09:44:06.321010Z 2021-12-10T09:44:06.229143Z 2021-12-10T09:44:05.720472Z 2021-12-10T09:44:05.470636Z 2021-12-10T09:44:05.345599Z 2021-12-10T09:44:05.040415Z 2021-12-10T09:44:04.883513Z 2021-12-10T09:44:04.651498Z 2021-12-10T09:44:03.922284Z 2021-12-10T09:44:02.533689Z 2021-12-10T09:44:02.513050Z 2021-12-10T09:44:01.939911Z 2021-12-10T09:44:01.628388Z 2021-12-10T09:44:01.473336Z 2021-12-10T09:44:01.344307Z 2021-12-10T09:44:01.262661Z 2021-12-10T09:44:01.244751Z 2021-12-10T09:44:00.976161Z 2021-12-10T09:44:00.903532Z 2021-12-10T09:44:00.860717Z 2021-12-10T09:44:00.806409Z 2021-12-10T09:44:00.763766Z 2021-12-10T09:44:00.182360Z 2021-12-10T09:44:00.147000Z 2021-12-10T09:43:59.932250Z 2021-12-10T09:43:59.776357Z 2021-12-10T09:43:59.642669Z 2021-12-10T09:43:59.320126Z 2021-12-10T09:43:59.211519Z 2021-12-10T09:43:59.052542Z 2021-12-10T09:43:59.035256Z 2021-12-10T09:43:58.647344Z 2021-12-10T09:43:58.548528Z 2021-12-10T09:43:58.318033Z 2021-12-10T09:43:58.050007Z 2021-12-10T09:43:57.937577Z 2021-12-10T09:43:57.907453Z 2021-12-10T09:43:57.408592Z 2021-12-10T09:43:57.361518Z 2021-12-10T09:43:57.311381Z 2021-12-10T09:43:57.149121Z 2021-12-10T09:43:57.004046Z 2021-12-10T09:43:56.959382Z 2021-12-10T09:43:56.928791Z 2021-12-10T09:43:56.869805Z 2021-12-10T09:43:56.463057Z 2021-12-10T09:43:56.384985Z 2021-12-10T09:43:56.322673Z 2021-12-10T09:43:56.148710Z 2021-12-10T09:43:56.114376Z 2021-12-10T09:43:55.941505Z 2021-12-10T09:43:55.915258Z 2021-12-10T09:43:55.821840Z 2021-12-10T09:43:55.592251Z 2021-12-10T09:43:55.392773Z 2021-12-10T09:43:55.274689Z 2021-12-10T09:43:55.093196Z 2021-12-10T09:43:54.962723Z 2021-12-10T09:43:54.904778Z 2021-12-10T09:43:54.895316Z 2021-12-10T09:43:54.854481Z 2021-12-10T09:43:54.698260Z 2021-12-10T09:43:54.409427Z 2021-12-10T09:43:54.370247Z 2021-12-10T09:43:54.359945Z 2021-12-10T09:43:54.333852Z 2021-12-10T09:43:54.180891Z 2021-12-10T09:43:54.146864Z 2021-12-10T09:43:54.058172Z 2021-12-10T09:43:53.745377Z 2021-12-10T09:43:53.680714Z 2021-12-10T09:43:53.668203Z 2021-12-10T09:43:53.555265Z 2021-12-10T09:43:53.552917Z 2021-12-10T09:43:53.526815Z 2021-12-10T09:43:53.503212Z 2021-12-10T09:43:53.133531Z 2021-12-10T09:43:53.101048Z 2021-12-10T09:43:52.980787Z 2021-12-10T09:43:52.903123Z 2021-12-10T09:43:52.885720Z 2021-12-10T09:43:52.722176Z 2021-12-10T09:43:52.708267Z 2021-12-10T09:43:52.034138Z 2021-12-10T09:48:18.418962Z 2021-12-10T09:48:17.973369Z 2021-12-10T09:48:17.608939Z 2021-12-10T09:48:17.529118Z 2021-12-10T09:48:17.356630Z 2021-12-10T09:48:16.896997Z 2021-12-10T09:48:16.890035Z 2021-12-10T09:48:16.716981Z 2021-12-10T09:48:16.493864Z 2021-12-10T09:48:16.184849Z 2021-12-10T09:48:15.890976Z 2021-12-10T09:48:15.875301Z 2021-12-10T09:48:15.811956Z 2021-12-10T09:48:15.807605Z 2021-12-10T09:48:15.800343Z 2021-12-10T09:48:15.700112Z 2021-12-10T09:48:15.584599Z 2021-12-10T09:48:15.575697Z 2021-12-10T09:48:15.572419Z 2021-12-10T09:48:15.482838Z 2021-12-10T09:48:15.151255Z 2021-12-10T09:48:14.985397Z 2021-12-10T09:48:14.779249Z 2021-12-10T09:48:14.616280Z 2021-12-10T09:48:14.431339Z 2021-12-10T09:48:14.413453Z 2021-12-10T09:48:14.240148Z 2021-12-10T09:48:14.146281Z 2021-12-10T09:48:14.065199Z 2021-12-10T09:48:13.927951Z 2021-12-10T09:48:13.869429Z 2021-12-10T09:48:13.625028Z 2021-12-10T09:48:13.597864Z 2021-12-10T09:48:13.401625Z 2021-12-10T09:48:13.242886Z 2021-12-10T09:48:13.156423Z 2021-12-10T09:48:13.150424Z 2021-12-10T09:48:13.123683Z 2021-12-10T09:48:13.076073Z 2021-12-10T09:48:12.967400Z 2021-12-10T09:48:12.819331Z 2021-12-10T09:48:12.815713Z 2021-12-10T09:48:12.556133Z 2021-12-10T09:48:12.286713Z 2021-12-10T09:48:12.178796Z 2021-12-10T09:48:12.043611Z 2021-12-10T09:48:12.037543Z 2021-12-10T09:48:12.012638Z 2021-12-10T09:48:11.905866Z 2021-12-10T09:48:11.904365Z 2021-12-10T09:48:11.761025Z 2021-12-10T09:48:11.533962Z 2021-12-10T09:48:11.452538Z 2021-12-10T09:48:11.245130Z 2021-12-10T09:48:11.217898Z 2021-12-10T09:48:11.142661Z 2021-12-10T09:48:11.130296Z 2021-12-10T09:48:11.022399Z 2021-12-10T09:48:10.732119Z 2021-12-10T09:48:10.587532Z 2021-12-10T09:48:10.532030Z 2021-12-10T09:48:10.392376Z 2021-12-10T09:48:10.340755Z 2021-12-10T09:48:10.136273Z 2021-12-10T09:48:10.109979Z 2021-12-10T09:48:09.832679Z 2021-12-10T09:48:09.811442Z 2021-12-10T09:48:09.761308Z 2021-12-10T09:48:09.663040Z 2021-12-10T09:48:09.565267Z 2021-12-10T09:48:09.523771Z 2021-12-10T09:48:09.505691Z 2021-12-10T09:48:09.391746Z 2021-12-10T09:48:09.321717Z 2021-12-10T09:48:09.270104Z 2021-12-10T09:48:09.245662Z 2021-12-10T09:48:09.213550Z 2021-12-10T09:48:09.166194Z 2021-12-10T09:48:09.045281Z 2021-12-10T09:48:08.999563Z 2021-12-10T09:48:08.940627Z 2021-12-10T09:48:08.918180Z 2021-12-10T09:48:08.320907Z 2021-12-10T09:48:08.314872Z 2021-12-10T09:48:08.277169Z 2021-12-10T09:48:08.225411Z 2021-12-10T09:48:08.110954Z 2021-12-10T09:48:08.101797Z 2021-12-10T09:48:08.089997Z 2021-12-10T09:48:08.043856Z 2021-12-10T09:48:08.042007Z 2021-12-10T09:48:08.019158Z 2021-12-10T09:48:07.995855Z 2021-12-10T09:48:07.864294Z 2021-12-10T09:48:07.729296Z 2021-12-10T09:48:07.457347Z 2021-12-10T09:48:07.441518Z 2021-12-10T09:48:07.391653Z 2021-12-10T09:48:07.333675Z 2021-12-10T09:48:06.318281Z 2021-12-10T09:52:24.687806Z 2021-12-10T09:52:24.230310Z 2021-12-10T09:52:23.657916Z 2021-12-10T09:52:23.433351Z 2021-12-10T09:52:23.229040Z 2021-12-10T09:52:23.055682Z 2021-12-10T09:52:23.046648Z 2021-12-10T09:52:23.041814Z 2021-12-10T09:52:23.004821Z 2021-12-10T09:52:22.877528Z 2021-12-10T09:52:22.827045Z 2021-12-10T09:52:22.598585Z 2021-12-10T09:52:22.541782Z 2021-12-10T09:52:22.376672Z 2021-12-10T09:52:22.042680Z 2021-12-10T09:52:21.926207Z 2021-12-10T09:52:21.921185Z 2021-12-10T09:52:21.843937Z 2021-12-10T09:52:21.743497Z 2021-12-10T09:52:21.469595Z 2021-12-10T09:52:21.461859Z 2021-12-10T09:52:21.462202Z 2021-12-10T09:52:21.308647Z 2021-12-10T09:52:21.213362Z 2021-12-10T09:52:21.206401Z 2021-12-10T09:52:21.013357Z 2021-12-10T09:52:20.851294Z 2021-12-10T09:52:20.840074Z 2021-12-10T09:52:20.553504Z 2021-12-10T09:52:20.420825Z 2021-12-10T09:52:20.407521Z 2021-12-10T09:52:20.384389Z 2021-12-10T09:52:20.355016Z 2021-12-10T09:52:20.021313Z 2021-12-10T09:52:19.947639Z 2021-12-10T09:52:19.872182Z 2021-12-10T09:52:19.837701Z 2021-12-10T09:52:19.798774Z 2021-12-10T09:52:19.612183Z 2021-12-10T09:52:19.612964Z 2021-12-10T09:52:19.508318Z 2021-12-10T09:52:19.375989Z 2021-12-10T09:52:19.180644Z 2021-12-10T09:52:19.095275Z 2021-12-10T09:52:18.825232Z 2021-12-10T09:52:18.747300Z 2021-12-10T09:52:18.738410Z 2021-12-10T09:52:18.707698Z 2021-12-10T09:52:18.517917Z 2021-12-10T09:52:18.392164Z 2021-12-10T09:52:18.385362Z 2021-12-10T09:52:18.283228Z 2021-12-10T09:52:18.263137Z 2021-12-10T09:52:18.238730Z 2021-12-10T09:52:18.003889Z 2021-12-10T09:52:17.999589Z 2021-12-10T09:52:17.948034Z 2021-12-10T09:52:17.915838Z 2021-12-10T09:52:17.880460Z 2021-12-10T09:52:17.764402Z 2021-12-10T09:52:17.689273Z 2021-12-10T09:52:17.671333Z 2021-12-10T09:52:17.654284Z 2021-12-10T09:52:17.614221Z 2021-12-10T09:52:17.319037Z 2021-12-10T09:52:17.287431Z 2021-12-10T09:52:17.203728Z 2021-12-10T09:52:17.039586Z 2021-12-10T09:52:17.020957Z 2021-12-10T09:52:16.916882Z 2021-12-10T09:52:16.894130Z 2021-12-10T09:52:16.881142Z 2021-12-10T09:52:16.803488Z 2021-12-10T09:52:16.769827Z 2021-12-10T09:52:16.661726Z 2021-12-10T09:52:16.610853Z 2021-12-10T09:52:16.355333Z 2021-12-10T09:52:16.358070Z 2021-12-10T09:52:16.269930Z 2021-12-10T09:52:16.198567Z 2021-12-10T09:52:16.167647Z 2021-12-10T09:52:16.135266Z 2021-12-10T09:52:16.077545Z 2021-12-10T09:52:16.050898Z 2021-12-10T09:52:16.023177Z 2021-12-10T09:52:16.012646Z 2021-12-10T09:52:15.973893Z 2021-12-10T09:52:15.883801Z 2021-12-10T09:52:15.815855Z 2021-12-10T09:52:15.791560Z 2021-12-10T09:52:15.790439Z 2021-12-10T09:52:15.516120Z 2021-12-10T09:52:15.369528Z 2021-12-10T09:52:15.349323Z 2021-12-10T09:52:15.208207Z 2021-12-10T09:52:15.078364Z 2021-12-10T09:52:15.062345Z 2021-12-10T09:52:15.041031Z 2021-12-10T09:52:14.926932Z 2021-12-10T09:52:14.923442Z 2021-12-10T09:56:42.366178Z 2021-12-10T09:56:42.168447Z 2021-12-10T09:56:41.734851Z 2021-12-10T09:56:40.998954Z 2021-12-10T09:56:41.009255Z 2021-12-10T09:56:40.850535Z 2021-12-10T09:56:40.772480Z 2021-12-10T09:56:40.695518Z 2021-12-10T09:56:40.469010Z 2021-12-10T09:56:40.383704Z 2021-12-10T09:56:40.184535Z 2021-12-10T09:56:40.000191Z 2021-12-10T09:56:39.693686Z 2021-12-10T09:56:39.377151Z 2021-12-10T09:56:39.262114Z 2021-12-10T09:56:39.172915Z 2021-12-10T09:56:39.158530Z 2021-12-10T09:56:38.939352Z 2021-12-10T09:56:38.765296Z 2021-12-10T09:56:38.519225Z 2021-12-10T09:56:38.505651Z 2021-12-10T09:56:38.500524Z 2021-12-10T09:56:38.396020Z 2021-12-10T09:56:38.350874Z 2021-12-10T09:56:38.157341Z 2021-12-10T09:56:38.121699Z 2021-12-10T09:56:38.016457Z 2021-12-10T09:56:37.782098Z 2021-12-10T09:56:37.732792Z 2021-12-10T09:56:37.629172Z 2021-12-10T09:56:37.581615Z 2021-12-10T09:56:37.523149Z 2021-12-10T09:56:37.457969Z 2021-12-10T09:56:37.349466Z 2021-12-10T09:56:37.312888Z 2021-12-10T09:56:36.988941Z 2021-12-10T09:56:36.967678Z 2021-12-10T09:56:36.842481Z 2021-12-10T09:56:36.756338Z 2021-12-10T09:56:36.744201Z 2021-12-10T09:56:36.740988Z 2021-12-10T09:56:36.657466Z 2021-12-10T09:56:36.090566Z 2021-12-10T09:56:35.893533Z 2021-12-10T09:56:35.770449Z 2021-12-10T09:56:35.642376Z 2021-12-10T09:56:35.482452Z 2021-12-10T09:56:35.470325Z 2021-12-10T09:56:35.463886Z 2021-12-10T09:56:35.427132Z 2021-12-10T09:56:35.373390Z 2021-12-10T09:56:35.073283Z 2021-12-10T09:56:35.028428Z 2021-12-10T09:56:34.928999Z 2021-12-10T09:56:34.858168Z 2021-12-10T09:56:34.801592Z 2021-12-10T09:56:34.651470Z 2021-12-10T09:56:34.647893Z 2021-12-10T09:56:34.630804Z 2021-12-10T09:56:34.605197Z 2021-12-10T09:56:34.584815Z 2021-12-10T09:56:34.549011Z 2021-12-10T09:56:34.453208Z 2021-12-10T09:56:34.312341Z 2021-12-10T09:56:34.239318Z 2021-12-10T09:56:34.198827Z 2021-12-10T09:56:33.749094Z 2021-12-10T09:56:33.647933Z 2021-12-10T09:56:33.598568Z 2021-12-10T09:56:33.252097Z 2021-12-10T09:56:33.046235Z 2021-12-10T09:56:33.036678Z 2021-12-10T09:56:32.996437Z 2021-12-10T09:56:32.976507Z 2021-12-10T09:56:32.970569Z 2021-12-10T09:56:32.957822Z 2021-12-10T09:56:32.933987Z 2021-12-10T09:56:32.878675Z 2021-12-10T09:56:32.735092Z 2021-12-10T09:56:32.689243Z 2021-12-10T09:56:32.661052Z 2021-12-10T09:56:32.631035Z 2021-12-10T09:56:32.561517Z 2021-12-10T09:56:32.507237Z 2021-12-10T09:56:32.432368Z 2021-12-10T09:56:32.369829Z 2021-12-10T09:56:32.329503Z 2021-12-10T09:56:32.190596Z 2021-12-10T09:56:32.159290Z 2021-12-10T09:56:32.066458Z 2021-12-10T09:56:31.846531Z 2021-12-10T09:56:31.678110Z 2021-12-10T09:56:31.609910Z 2021-12-10T09:56:31.546096Z 2021-12-10T09:56:31.520199Z 2021-12-10T09:56:31.484375Z 2021-12-10T09:56:31.429964Z 2021-12-10T09:56:31.302658Z 2021-12-10T09:56:31.042630Z 2021-12-10T09:56:31.025113Z", + "created_diff": 16.8510366, + "ended": "2021-12-10T09:58:11.620046Z", + "error_job_ids": " [328]", + "event_first": "2021-12-10T09:40:18.272981Z 2021-12-10T09:39:36.870876Z 2021-12-10T09:40:12.373094Z 2021-12-10T09:39:37.639136Z 2021-12-10T09:40:19.036601Z 2021-12-10T09:39:29.840979Z 2021-12-10T09:40:20.068872Z 2021-12-10T09:39:29.286364Z 2021-12-10T09:40:17.220610Z 2021-12-10T09:39:31.311531Z 2021-12-10T09:39:55.436087Z 2021-12-10T09:40:58.810608Z 2021-12-10T09:39:50.383128Z 2021-12-10T09:40:50.080789Z 2021-12-10T09:39:57.389817Z 2021-12-10T09:41:45.895115Z 2021-12-10T09:39:51.260957Z 2021-12-10T09:40:31.799124Z 2021-12-10T09:39:52.377858Z 2021-12-10T09:41:36.793221Z 2021-12-10T09:39:56.551804Z 2021-12-10T09:41:08.219270Z 2021-12-10T09:39:48.737499Z 2021-12-10T09:40:41.642541Z 2021-12-10T09:41:27.808502Z 2021-12-10T09:39:43.886596Z 2021-12-10T09:39:23.321729Z 2021-12-10T09:41:17.898229Z 2021-12-10T09:39:21.760656Z 2021-12-10T09:40:16.441553Z 2021-12-10T09:39:21.617617Z 2021-12-10T09:41:55.971797Z 2021-12-10T09:39:11.229849Z 2021-12-10T09:39:24.608583Z 2021-12-10T09:39:04.725660Z 2021-12-10T09:40:26.241515Z 2021-12-10T09:40:36.216911Z 2021-12-10T09:40:23.609952Z 2021-12-10T09:40:37.987067Z 2021-12-10T09:40:25.286331Z 2021-12-10T09:40:27.146438Z 2021-12-10T09:40:26.965527Z 2021-12-10T09:40:21.005417Z 2021-12-10T09:40:26.447543Z 2021-12-10T09:40:38.507005Z 2021-12-10T09:40:22.319489Z 2021-12-10T09:40:33.804836Z 2021-12-10T09:40:21.273454Z 2021-12-10T09:40:34.050849Z 2021-12-10T09:40:20.551838Z 2021-12-10T09:40:34.797761Z 2021-12-10T09:40:19.996687Z 2021-12-10T09:40:32.768852Z 2021-12-10T09:40:20.813977Z 2021-12-10T09:40:36.217587Z 2021-12-10T09:40:16.433190Z 2021-12-10T09:40:35.490443Z 2021-12-10T09:40:16.433190Z 2021-12-10T09:40:34.547574Z 2021-12-10T09:40:15.609832Z 2021-12-10T09:40:35.051557Z 2021-12-10T09:40:14.592924Z 2021-12-10T09:40:32.759571Z 2021-12-10T09:40:13.322578Z 2021-12-10T09:40:33.797689Z 2021-12-10T09:40:11.575284Z 2021-12-10T09:40:27.919860Z 2021-12-10T09:40:21.816739Z 2021-12-10T09:40:35.449653Z 2021-12-10T09:40:19.770654Z 2021-12-10T09:40:27.582698Z 2021-12-10T09:40:16.692970Z 2021-12-10T09:40:30.707578Z 2021-12-10T09:40:13.132724Z 2021-12-10T09:40:30.445374Z 2021-12-10T09:40:08.327389Z 2021-12-10T09:40:27.666687Z 2021-12-10T09:39:40.761457Z 2021-12-10T09:40:29.181528Z 2021-12-10T09:40:30.364733Z 2021-12-10T09:40:17.850485Z 2021-12-10T09:39:51.784179Z 2021-12-10T09:40:20.468976Z 2021-12-10T09:39:43.591868Z 2021-12-10T09:40:28.930409Z 2021-12-10T09:39:47.202031Z 2021-12-10T09:40:13.476487Z 2021-12-10T09:39:40.342449Z 2021-12-10T09:40:25.097784Z 2021-12-10T09:39:44.942928Z 2021-12-10T09:40:20.983962Z 2021-12-10T09:39:43.591868Z 2021-12-10T09:40:13.476487Z 2021-12-10T09:39:41.272028Z 2021-12-10T09:40:12.803253Z 2021-12-10T09:39:39.984956Z 2021-12-10T09:40:16.135560Z 2021-12-10T09:39:35.646322Z 2021-12-10T09:40:13.927714Z 2021-12-10T09:39:42.048910Z 2021-12-10T09:45:28.776291Z 2021-12-10T09:45:33.710483Z 2021-12-10T09:45:30.597915Z 2021-12-10T09:45:28.814679Z 2021-12-10T09:45:30.791476Z 2021-12-10T09:45:28.871598Z 2021-12-10T09:45:25.948790Z 2021-12-10T09:45:29.811826Z 2021-12-10T09:45:27.926014Z 2021-12-10T09:45:32.408818Z 2021-12-10T09:45:26.428986Z 2021-12-10T09:45:31.159655Z 2021-12-10T09:45:27.418814Z 2021-12-10T09:45:28.065819Z 2021-12-10T09:45:27.671484Z 2021-12-10T09:45:32.365479Z 2021-12-10T09:45:25.665669Z 2021-12-10T09:45:30.058133Z 2021-12-10T09:45:25.296481Z 2021-12-10T09:45:26.248732Z 2021-12-10T09:45:27.161432Z 2021-12-10T09:45:29.065549Z 2021-12-10T09:45:27.671484Z 2021-12-10T09:45:30.087046Z 2021-12-10T09:45:29.303912Z 2021-12-10T09:45:16.107683Z 2021-12-10T09:45:21.877578Z 2021-12-10T09:45:20.237938Z 2021-12-10T09:45:22.348708Z 2021-12-10T09:45:21.345946Z 2021-12-10T09:45:24.413740Z 2021-12-10T09:45:18.459815Z 2021-12-10T09:45:16.346186Z 2021-12-10T09:45:20.237938Z 2021-12-10T09:45:22.863094Z 2021-12-10T09:45:25.057305Z 2021-12-10T09:45:30.818173Z 2021-12-10T09:45:16.778811Z 2021-12-10T09:45:22.286987Z 2021-12-10T09:45:20.418830Z 2021-12-10T09:45:16.412758Z 2021-12-10T09:45:20.921026Z 2021-12-10T09:45:25.361907Z 2021-12-10T09:45:09.068015Z 2021-12-10T09:45:20.660435Z 2021-12-10T09:45:07.880798Z 2021-12-10T09:45:15.354306Z 2021-12-10T09:45:20.675118Z 2021-12-10T09:45:10.520801Z 2021-12-10T09:45:20.675063Z 2021-12-10T09:45:10.104852Z 2021-12-10T09:45:15.092762Z 2021-12-10T09:45:09.945390Z 2021-12-10T09:45:13.287342Z 2021-12-10T09:45:17.189481Z 2021-12-10T09:45:14.742649Z 2021-12-10T09:45:08.772086Z 2021-12-10T09:45:13.738674Z 2021-12-10T09:45:13.299557Z 2021-12-10T09:45:09.991219Z 2021-12-10T09:45:05.090357Z 2021-12-10T09:45:07.885654Z 2021-12-10T09:45:09.051690Z 2021-12-10T09:45:07.302138Z 2021-12-10T09:45:03.083826Z 2021-12-10T09:44:52.378759Z 2021-12-10T09:45:11.390519Z 2021-12-10T09:45:02.549976Z 2021-12-10T09:44:58.227336Z 2021-12-10T09:45:03.939150Z 2021-12-10T09:44:55.626458Z 2021-12-10T09:44:55.988663Z 2021-12-10T09:45:04.631529Z 2021-12-10T09:44:58.719797Z 2021-12-10T09:44:52.149041Z 2021-12-10T09:44:51.070431Z 2021-12-10T09:44:55.284853Z 2021-12-10T09:44:43.733811Z 2021-12-10T09:44:27.288843Z 2021-12-10T09:44:20.596572Z 2021-12-10T09:44:29.453396Z 2021-12-10T09:44:20.535855Z 2021-12-10T09:44:21.964588Z 2021-12-10T09:44:23.232916Z 2021-12-10T09:44:23.480492Z 2021-12-10T09:44:19.778603Z 2021-12-10T09:44:16.627561Z 2021-12-10T09:44:16.951725Z 2021-12-10T09:44:20.602723Z 2021-12-10T09:44:16.694817Z 2021-12-10T09:44:16.972582Z 2021-12-10T09:44:15.933700Z 2021-12-10T09:44:13.755262Z 2021-12-10T09:44:15.210599Z 2021-12-10T09:44:15.269877Z 2021-12-10T09:44:10.501663Z 2021-12-10T09:44:04.848022Z 2021-12-10T09:44:05.257428Z 2021-12-10T09:44:01.563463Z 2021-12-10T09:43:59.612825Z 2021-12-10T09:49:43.482917Z 2021-12-10T09:49:45.466508Z 2021-12-10T09:49:45.262840Z 2021-12-10T09:49:42.091574Z 2021-12-10T09:49:44.497700Z 2021-12-10T09:49:41.663446Z 2021-12-10T09:49:45.565095Z 2021-12-10T09:49:43.075526Z 2021-12-10T09:49:45.262840Z 2021-12-10T09:49:38.192502Z 2021-12-10T09:49:43.482917Z 2021-12-10T09:49:43.302573Z 2021-12-10T09:49:44.299728Z 2021-12-10T09:49:35.072346Z 2021-12-10T09:49:47.847231Z 2021-12-10T09:49:36.788409Z 2021-12-10T09:49:38.398651Z 2021-12-10T09:49:30.427514Z 2021-12-10T09:49:40.979025Z 2021-12-10T09:49:43.302573Z 2021-12-10T09:49:37.230621Z 2021-12-10T09:49:41.333259Z 2021-12-10T09:49:42.228158Z 2021-12-10T09:49:41.594696Z 2021-12-10T09:49:39.163621Z 2021-12-10T09:49:44.333688Z 2021-12-10T09:49:43.482917Z 2021-12-10T09:49:38.901749Z 2021-12-10T09:49:36.981696Z 2021-12-10T09:49:38.134539Z 2021-12-10T09:49:41.980791Z 2021-12-10T09:49:37.275526Z 2021-12-10T09:49:41.741661Z 2021-12-10T09:49:40.216212Z 2021-12-10T09:49:38.581685Z 2021-12-10T09:49:34.977940Z 2021-12-10T09:49:36.680638Z 2021-12-10T09:49:34.600055Z 2021-12-10T09:49:38.728386Z 2021-12-10T09:49:32.761587Z 2021-12-10T09:49:31.057661Z 2021-12-10T09:49:33.165468Z 2021-12-10T09:49:35.511863Z 2021-12-10T09:49:34.369239Z 2021-12-10T09:49:35.648753Z 2021-12-10T09:49:36.408819Z 2021-12-10T09:49:30.928004Z 2021-12-10T09:49:28.861098Z 2021-12-10T09:49:35.211689Z 2021-12-10T09:49:27.349625Z 2021-12-10T09:49:35.208770Z 2021-12-10T09:49:33.582869Z 2021-12-10T09:49:31.421980Z 2021-12-10T09:49:26.185856Z 2021-12-10T09:49:25.035286Z 2021-12-10T09:49:26.604768Z 2021-12-10T09:49:30.686939Z 2021-12-10T09:49:29.904557Z 2021-12-10T09:49:17.254731Z 2021-12-10T09:49:27.391739Z 2021-12-10T09:49:10.334516Z 2021-12-10T09:49:29.141542Z 2021-12-10T09:49:23.421952Z 2021-12-10T09:49:18.689584Z 2021-12-10T09:49:15.857473Z 2021-12-10T09:49:15.000092Z 2021-12-10T09:49:09.559628Z 2021-12-10T09:49:12.761381Z 2021-12-10T09:49:07.835089Z 2021-12-10T09:49:05.637634Z 2021-12-10T09:49:16.145176Z 2021-12-10T09:49:08.052620Z 2021-12-10T09:49:01.739016Z 2021-12-10T09:49:03.296627Z 2021-12-10T09:49:06.607784Z 2021-12-10T09:49:00.978535Z 2021-12-10T09:48:59.006483Z 2021-12-10T09:48:55.372841Z 2021-12-10T09:49:04.181764Z 2021-12-10T09:49:02.181738Z 2021-12-10T09:49:05.334717Z 2021-12-10T09:48:56.591447Z 2021-12-10T09:48:51.834716Z 2021-12-10T09:48:50.282890Z 2021-12-10T09:48:42.922510Z 2021-12-10T09:48:45.175998Z 2021-12-10T09:48:53.581229Z 2021-12-10T09:48:39.048865Z 2021-12-10T09:48:47.403544Z 2021-12-10T09:48:38.901976Z 2021-12-10T09:48:42.075746Z 2021-12-10T09:48:34.521405Z 2021-12-10T09:48:36.644077Z 2021-12-10T09:48:21.405807Z 2021-12-10T09:48:23.287157Z 2021-12-10T09:48:19.936307Z 2021-12-10T09:48:20.100097Z 2021-12-10T09:48:17.788366Z 2021-12-10T09:48:13.800563Z 2021-12-10T09:53:48.219211Z 2021-12-10T09:53:53.903750Z 2021-12-10T09:53:51.495478Z 2021-12-10T09:53:54.968828Z 2021-12-10T09:53:51.871408Z 2021-12-10T09:53:53.397731Z 2021-12-10T09:53:49.173800Z 2021-12-10T09:53:51.602624Z 2021-12-10T09:53:46.052640Z 2021-12-10T09:53:51.693699Z 2021-12-10T09:53:49.402991Z 2021-12-10T09:53:53.652857Z 2021-12-10T09:53:48.219211Z 2021-12-10T09:53:51.234645Z 2021-12-10T09:53:47.317690Z 2021-12-10T09:53:51.180664Z 2021-12-10T09:53:44.388448Z 2021-12-10T09:53:56.545400Z 2021-12-10T09:53:44.348562Z 2021-12-10T09:53:53.951967Z 2021-12-10T09:53:52.395668Z 2021-12-10T09:53:47.681149Z 2021-12-10T09:53:47.681149Z 2021-12-10T09:53:52.227692Z 2021-12-10T09:53:43.631198Z 2021-12-10T09:53:46.809863Z 2021-12-10T09:53:49.955750Z 2021-12-10T09:53:56.044743Z 2021-12-10T09:53:42.723594Z 2021-12-10T09:53:47.583913Z 2021-12-10T09:53:40.115458Z 2021-12-10T09:53:44.825608Z 2021-12-10T09:53:37.326676Z 2021-12-10T09:53:46.053169Z 2021-12-10T09:53:43.897793Z 2021-12-10T09:53:47.817462Z 2021-12-10T09:53:45.156278Z 2021-12-10T09:53:46.306945Z 2021-12-10T09:53:47.332882Z 2021-12-10T09:53:49.402991Z 2021-12-10T09:53:46.052640Z 2021-12-10T09:53:48.826170Z 2021-12-10T09:53:44.871645Z 2021-12-10T09:53:49.968941Z 2021-12-10T09:53:39.098466Z 2021-12-10T09:53:43.433275Z 2021-12-10T09:53:38.185909Z 2021-12-10T09:53:38.026010Z 2021-12-10T09:53:33.387448Z 2021-12-10T09:53:42.823497Z 2021-12-10T09:53:40.115458Z 2021-12-10T09:53:43.562811Z 2021-12-10T09:53:36.586164Z 2021-12-10T09:53:38.176489Z 2021-12-10T09:53:26.274875Z 2021-12-10T09:53:34.515715Z 2021-12-10T09:53:24.658639Z 2021-12-10T09:53:16.407360Z 2021-12-10T09:53:20.474712Z 2021-12-10T09:53:25.571726Z 2021-12-10T09:53:25.188559Z 2021-12-10T09:53:28.888872Z 2021-12-10T09:53:18.187670Z 2021-12-10T09:53:21.972863Z 2021-12-10T09:53:16.714343Z 2021-12-10T09:53:31.680578Z 2021-12-10T09:53:07.490681Z 2021-12-10T09:53:22.539299Z 2021-12-10T09:53:11.901515Z 2021-12-10T09:53:28.901475Z 2021-12-10T09:53:21.324985Z 2021-12-10T09:53:27.477701Z 2021-12-10T09:53:13.109283Z 2021-12-10T09:53:16.524679Z 2021-12-10T09:53:16.081564Z 2021-12-10T09:53:21.490883Z 2021-12-10T09:53:15.478042Z 2021-12-10T09:53:12.853122Z 2021-12-10T09:53:08.274469Z 2021-12-10T09:53:19.481680Z 2021-12-10T09:52:58.806281Z 2021-12-10T09:53:02.476261Z 2021-12-10T09:52:56.992550Z 2021-12-10T09:52:59.561057Z 2021-12-10T09:52:57.063016Z 2021-12-10T09:53:11.268439Z 2021-12-10T09:52:52.727281Z 2021-12-10T09:53:00.031352Z 2021-12-10T09:52:49.461386Z 2021-12-10T09:52:58.476674Z 2021-12-10T09:52:44.451472Z 2021-12-10T09:52:52.715572Z 2021-12-10T09:52:39.327223Z 2021-12-10T09:52:55.992696Z 2021-12-10T09:52:27.953478Z 2021-12-10T09:52:30.351136Z 2021-12-10T09:52:26.475477Z 2021-12-10T09:52:27.103966Z 2021-12-10T09:52:26.138707Z 2021-12-10T09:52:26.175568Z 2021-12-10T09:58:07.055244Z 2021-12-10T09:58:08.539255Z 2021-12-10T09:58:08.303636Z 2021-12-10T09:58:04.592685Z 2021-12-10T09:58:03.306958Z 2021-12-10T09:58:08.036650Z 2021-12-10T09:58:05.350470Z 2021-12-10T09:58:08.058525Z 2021-12-10T09:58:09.685140Z 2021-12-10T09:58:09.003433Z 2021-12-10T09:58:08.533439Z 2021-12-10T09:58:05.334688Z 2021-12-10T09:58:08.430804Z 2021-12-10T09:58:06.773416Z 2021-12-10T09:58:06.279671Z 2021-12-10T09:58:06.528250Z 2021-12-10T09:58:05.350470Z 2021-12-10T09:58:06.064442Z 2021-12-10T09:58:07.296975Z 2021-12-10T09:58:04.310455Z 2021-12-10T09:58:07.885185Z 2021-12-10T09:58:06.316788Z 2021-12-10T09:58:06.871147Z 2021-12-10T09:58:02.532142Z 2021-12-10T09:58:05.350470Z 2021-12-10T09:58:03.306958Z 2021-12-10T09:57:59.077649Z 2021-12-10T09:58:03.050671Z 2021-12-10T09:58:01.566785Z 2021-12-10T09:58:02.046430Z 2021-12-10T09:58:06.606629Z 2021-12-10T09:58:08.036650Z 2021-12-10T09:58:01.208732Z 2021-12-10T09:58:04.806736Z 2021-12-10T09:58:03.334723Z 2021-12-10T09:58:04.563712Z 2021-12-10T09:58:07.055244Z 2021-12-10T09:58:01.556410Z 2021-12-10T09:57:59.578660Z 2021-12-10T09:58:01.308726Z 2021-12-10T09:57:50.577321Z 2021-12-10T09:58:01.798723Z 2021-12-10T09:57:48.969472Z 2021-12-10T09:57:57.860934Z 2021-12-10T09:57:57.463037Z 2021-12-10T09:57:43.396705Z 2021-12-10T09:57:56.995537Z 2021-12-10T09:57:49.377572Z 2021-12-10T09:57:53.025496Z 2021-12-10T09:57:47.034726Z 2021-12-10T09:57:47.690157Z 2021-12-10T09:57:51.981686Z 2021-12-10T09:57:55.203421Z 2021-12-10T09:57:49.714010Z 2021-12-10T09:57:49.219911Z 2021-12-10T09:57:44.603694Z 2021-12-10T09:57:51.993729Z 2021-12-10T09:57:44.316892Z 2021-12-10T09:57:48.010621Z 2021-12-10T09:57:44.316892Z 2021-12-10T09:57:42.481493Z 2021-12-10T09:57:51.318386Z 2021-12-10T09:57:47.531747Z 2021-12-10T09:57:43.269743Z 2021-12-10T09:57:39.599390Z 2021-12-10T09:57:39.813621Z 2021-12-10T09:57:40.444436Z 2021-12-10T09:57:45.310462Z 2021-12-10T09:57:41.937130Z 2021-12-10T09:57:31.726754Z 2021-12-10T09:57:39.641995Z 2021-12-10T09:57:34.894544Z 2021-12-10T09:57:39.730145Z 2021-12-10T09:57:37.965475Z 2021-12-10T09:57:31.719625Z 2021-12-10T09:57:37.710741Z 2021-12-10T09:57:32.615399Z 2021-12-10T09:57:22.057999Z 2021-12-10T09:57:39.269287Z 2021-12-10T09:57:22.149858Z 2021-12-10T09:57:27.278866Z 2021-12-10T09:57:33.436599Z 2021-12-10T09:57:25.010723Z 2021-12-10T09:57:31.560790Z 2021-12-10T09:57:22.402472Z 2021-12-10T09:57:23.611814Z 2021-12-10T09:57:25.926586Z 2021-12-10T09:57:24.684191Z 2021-12-10T09:57:05.274279Z 2021-12-10T09:57:26.793943Z 2021-12-10T09:56:54.643138Z 2021-12-10T09:56:59.800657Z 2021-12-10T09:56:49.606904Z 2021-12-10T09:56:56.679509Z 2021-12-10T09:56:53.602542Z 2021-12-10T09:56:47.219945Z 2021-12-10T09:56:45.155447Z 2021-12-10T09:56:44.640175Z 2021-12-10T09:56:43.762046Z 2021-12-10T09:56:43.800445Z", + "event_last": "2021-12-10T09:40:21.992183Z 2021-12-10T09:39:39.845955Z 2021-12-10T09:40:16.265405Z 2021-12-10T09:39:40.885295Z 2021-12-10T09:40:22.124549Z 2021-12-10T09:39:31.852865Z 2021-12-10T09:40:23.809911Z 2021-12-10T09:39:31.341399Z 2021-12-10T09:40:21.694674Z 2021-12-10T09:39:33.297264Z 2021-12-10T09:39:59.147655Z 2021-12-10T09:40:59.194110Z 2021-12-10T09:39:55.048426Z 2021-12-10T09:40:50.509057Z 2021-12-10T09:40:01.925465Z 2021-12-10T09:41:46.717469Z 2021-12-10T09:39:56.298530Z 2021-12-10T09:40:32.563390Z 2021-12-10T09:39:58.638866Z 2021-12-10T09:41:37.583365Z 2021-12-10T09:40:01.219442Z 2021-12-10T09:41:09.595848Z 2021-12-10T09:39:52.093968Z 2021-12-10T09:40:43.889539Z 2021-12-10T09:41:27.972577Z 2021-12-10T09:39:48.999999Z 2021-12-10T09:39:26.008483Z 2021-12-10T09:41:18.267977Z 2021-12-10T09:39:24.584895Z 2021-12-10T09:40:19.281746Z 2021-12-10T09:39:24.986409Z 2021-12-10T09:41:56.392922Z 2021-12-10T09:39:13.143976Z 2021-12-10T09:39:26.073277Z 2021-12-10T09:39:06.141435Z 2021-12-10T09:40:27.489486Z 2021-12-10T09:40:38.585468Z 2021-12-10T09:40:25.291673Z 2021-12-10T09:40:39.480723Z 2021-12-10T09:40:26.870160Z 2021-12-10T09:40:28.202675Z 2021-12-10T09:40:27.815231Z 2021-12-10T09:40:23.777309Z 2021-12-10T09:40:27.729969Z 2021-12-10T09:40:39.565479Z 2021-12-10T09:40:24.830904Z 2021-12-10T09:40:36.927603Z 2021-12-10T09:40:23.584795Z 2021-12-10T09:40:36.067253Z 2021-12-10T09:40:23.750391Z 2021-12-10T09:40:36.958571Z 2021-12-10T09:40:23.329688Z 2021-12-10T09:40:35.320469Z 2021-12-10T09:40:24.067003Z 2021-12-10T09:40:38.328720Z 2021-12-10T09:40:20.019748Z 2021-12-10T09:40:38.357516Z 2021-12-10T09:40:19.811333Z 2021-12-10T09:40:36.664771Z 2021-12-10T09:40:18.137301Z 2021-12-10T09:40:37.746720Z 2021-12-10T09:40:16.513189Z 2021-12-10T09:40:35.577243Z 2021-12-10T09:40:16.076317Z 2021-12-10T09:40:37.075306Z 2021-12-10T09:40:14.687893Z 2021-12-10T09:40:30.714521Z 2021-12-10T09:40:24.523411Z 2021-12-10T09:40:37.830372Z 2021-12-10T09:40:22.433148Z 2021-12-10T09:40:30.943097Z 2021-12-10T09:40:19.811333Z 2021-12-10T09:40:33.524416Z 2021-12-10T09:40:15.465314Z 2021-12-10T09:40:34.396629Z 2021-12-10T09:40:10.551310Z 2021-12-10T09:40:31.148338Z 2021-12-10T09:39:44.088249Z 2021-12-10T09:40:32.903400Z 2021-12-10T09:40:33.292300Z 2021-12-10T09:40:22.830244Z 2021-12-10T09:39:55.839857Z 2021-12-10T09:40:24.912083Z 2021-12-10T09:39:47.871344Z 2021-12-10T09:40:33.049918Z 2021-12-10T09:39:50.466388Z 2021-12-10T09:40:16.820408Z 2021-12-10T09:39:44.432174Z 2021-12-10T09:40:27.477962Z 2021-12-10T09:39:48.943253Z 2021-12-10T09:40:25.559426Z 2021-12-10T09:39:47.052989Z 2021-12-10T09:40:17.308569Z 2021-12-10T09:39:44.767656Z 2021-12-10T09:40:16.035468Z 2021-12-10T09:39:43.537531Z 2021-12-10T09:40:20.735414Z 2021-12-10T09:39:38.226902Z 2021-12-10T09:40:17.812398Z 2021-12-10T09:39:46.562413Z 2021-12-10T09:45:30.903628Z 2021-12-10T09:45:35.991921Z 2021-12-10T09:45:31.612454Z 2021-12-10T09:45:30.523726Z 2021-12-10T09:45:31.565199Z 2021-12-10T09:45:30.979603Z 2021-12-10T09:45:28.200815Z 2021-12-10T09:45:32.001962Z 2021-12-10T09:45:30.033913Z 2021-12-10T09:45:33.673408Z 2021-12-10T09:45:28.861572Z 2021-12-10T09:45:33.177133Z 2021-12-10T09:45:29.378386Z 2021-12-10T09:45:29.300406Z 2021-12-10T09:45:29.810484Z 2021-12-10T09:45:33.791557Z 2021-12-10T09:45:28.416406Z 2021-12-10T09:45:31.405608Z 2021-12-10T09:45:27.966435Z 2021-12-10T09:45:28.551439Z 2021-12-10T09:45:29.021329Z 2021-12-10T09:45:31.261573Z 2021-12-10T09:45:29.581755Z 2021-12-10T09:45:32.471421Z 2021-12-10T09:45:32.241419Z 2021-12-10T09:45:18.894329Z 2021-12-10T09:45:25.385462Z 2021-12-10T09:45:24.055251Z 2021-12-10T09:45:25.625126Z 2021-12-10T09:45:24.184250Z 2021-12-10T09:45:27.138125Z 2021-12-10T09:45:22.041300Z 2021-12-10T09:45:21.008483Z 2021-12-10T09:45:23.270488Z 2021-12-10T09:45:26.294427Z 2021-12-10T09:45:27.357381Z 2021-12-10T09:45:33.123107Z 2021-12-10T09:45:21.273995Z 2021-12-10T09:45:25.057305Z 2021-12-10T09:45:23.731461Z 2021-12-10T09:45:20.334690Z 2021-12-10T09:45:24.678424Z 2021-12-10T09:45:27.690290Z 2021-12-10T09:45:13.266827Z 2021-12-10T09:45:24.166280Z 2021-12-10T09:45:11.746279Z 2021-12-10T09:45:18.036313Z 2021-12-10T09:45:24.411291Z 2021-12-10T09:45:13.781327Z 2021-12-10T09:45:24.655266Z 2021-12-10T09:45:13.058356Z 2021-12-10T09:45:18.505647Z 2021-12-10T09:45:14.039271Z 2021-12-10T09:45:16.609481Z 2021-12-10T09:45:20.788382Z 2021-12-10T09:45:19.406529Z 2021-12-10T09:45:12.768374Z 2021-12-10T09:45:17.974347Z 2021-12-10T09:45:16.710258Z 2021-12-10T09:45:13.023477Z 2021-12-10T09:45:09.079294Z 2021-12-10T09:45:11.233158Z 2021-12-10T09:45:12.755346Z 2021-12-10T09:45:11.123431Z 2021-12-10T09:45:06.937126Z 2021-12-10T09:44:57.045771Z 2021-12-10T09:45:15.041261Z 2021-12-10T09:45:05.714517Z 2021-12-10T09:45:01.357268Z 2021-12-10T09:45:07.730527Z 2021-12-10T09:44:59.411305Z 2021-12-10T09:45:00.012446Z 2021-12-10T09:45:07.992267Z 2021-12-10T09:45:02.002361Z 2021-12-10T09:44:55.959284Z 2021-12-10T09:44:54.792998Z 2021-12-10T09:44:59.411305Z 2021-12-10T09:44:47.403502Z 2021-12-10T09:44:29.687131Z 2021-12-10T09:44:22.105646Z 2021-12-10T09:44:32.852256Z 2021-12-10T09:44:22.387725Z 2021-12-10T09:44:23.983899Z 2021-12-10T09:44:25.650980Z 2021-12-10T09:44:25.640911Z 2021-12-10T09:44:21.326535Z 2021-12-10T09:44:18.024322Z 2021-12-10T09:44:18.646888Z 2021-12-10T09:44:22.591167Z 2021-12-10T09:44:18.292107Z 2021-12-10T09:44:18.664941Z 2021-12-10T09:44:17.495776Z 2021-12-10T09:44:14.779993Z 2021-12-10T09:44:16.245391Z 2021-12-10T09:44:16.433209Z 2021-12-10T09:44:11.507444Z 2021-12-10T09:44:06.565197Z 2021-12-10T09:44:06.635110Z 2021-12-10T09:44:02.761319Z 2021-12-10T09:44:01.121575Z 2021-12-10T09:49:45.737440Z 2021-12-10T09:49:47.771884Z 2021-12-10T09:49:47.564400Z 2021-12-10T09:49:44.269694Z 2021-12-10T09:49:46.655434Z 2021-12-10T09:49:44.022175Z 2021-12-10T09:49:47.988593Z 2021-12-10T09:49:44.799751Z 2021-12-10T09:49:47.492060Z 2021-12-10T09:49:42.099359Z 2021-12-10T09:49:45.930068Z 2021-12-10T09:49:45.389703Z 2021-12-10T09:49:46.660458Z 2021-12-10T09:49:39.003821Z 2021-12-10T09:49:49.640730Z 2021-12-10T09:49:39.350556Z 2021-12-10T09:49:41.841665Z 2021-12-10T09:49:33.459583Z 2021-12-10T09:49:44.550151Z 2021-12-10T09:49:44.913298Z 2021-12-10T09:49:40.423329Z 2021-12-10T09:49:43.899293Z 2021-12-10T09:49:45.169007Z 2021-12-10T09:49:44.192584Z 2021-12-10T09:49:42.951464Z 2021-12-10T09:49:46.009289Z 2021-12-10T09:49:46.129552Z 2021-12-10T09:49:41.910435Z 2021-12-10T09:49:39.988421Z 2021-12-10T09:49:41.227909Z 2021-12-10T09:49:45.737440Z 2021-12-10T09:49:40.697301Z 2021-12-10T09:49:43.883889Z 2021-12-10T09:49:42.351325Z 2021-12-10T09:49:41.169463Z 2021-12-10T09:49:38.901749Z 2021-12-10T09:49:40.174055Z 2021-12-10T09:49:39.152276Z 2021-12-10T09:49:42.623451Z 2021-12-10T09:49:36.743574Z 2021-12-10T09:49:34.154038Z 2021-12-10T09:49:37.349414Z 2021-12-10T09:49:38.313917Z 2021-12-10T09:49:37.497160Z 2021-12-10T09:49:40.330326Z 2021-12-10T09:49:39.896537Z 2021-12-10T09:49:33.176523Z 2021-12-10T09:49:33.423481Z 2021-12-10T09:49:39.574040Z 2021-12-10T09:49:31.057661Z 2021-12-10T09:49:39.804281Z 2021-12-10T09:49:37.678427Z 2021-12-10T09:49:34.826310Z 2021-12-10T09:49:29.631172Z 2021-12-10T09:49:27.736381Z 2021-12-10T09:49:30.595602Z 2021-12-10T09:49:34.080807Z 2021-12-10T09:49:34.126056Z 2021-12-10T09:49:21.873549Z 2021-12-10T09:49:29.648418Z 2021-12-10T09:49:14.356912Z 2021-12-10T09:49:32.310510Z 2021-12-10T09:49:26.764512Z 2021-12-10T09:49:22.111260Z 2021-12-10T09:49:20.158520Z 2021-12-10T09:49:18.567525Z 2021-12-10T09:49:13.712303Z 2021-12-10T09:49:16.358477Z 2021-12-10T09:49:12.425987Z 2021-12-10T09:49:10.291572Z 2021-12-10T09:49:20.776976Z 2021-12-10T09:49:12.498888Z 2021-12-10T09:49:06.579550Z 2021-12-10T09:49:09.333675Z 2021-12-10T09:49:11.701936Z 2021-12-10T09:49:05.063273Z 2021-12-10T09:49:03.170379Z 2021-12-10T09:48:59.903320Z 2021-12-10T09:49:08.444700Z 2021-12-10T09:49:06.987242Z 2021-12-10T09:49:10.448470Z 2021-12-10T09:49:00.344314Z 2021-12-10T09:48:57.168876Z 2021-12-10T09:48:56.070327Z 2021-12-10T09:48:47.306547Z 2021-12-10T09:48:49.031578Z 2021-12-10T09:48:59.746432Z 2021-12-10T09:48:42.571692Z 2021-12-10T09:48:52.128455Z 2021-12-10T09:48:42.072334Z 2021-12-10T09:48:45.417734Z 2021-12-10T09:48:36.908967Z 2021-12-10T09:48:39.748178Z 2021-12-10T09:48:23.420139Z 2021-12-10T09:48:24.407443Z 2021-12-10T09:48:20.678428Z 2021-12-10T09:48:20.915187Z 2021-12-10T09:48:19.936525Z 2021-12-10T09:48:14.931823Z 2021-12-10T09:53:50.633594Z 2021-12-10T09:53:56.478173Z 2021-12-10T09:53:52.824697Z 2021-12-10T09:53:56.668850Z 2021-12-10T09:53:53.001416Z 2021-12-10T09:53:56.281413Z 2021-12-10T09:53:51.254539Z 2021-12-10T09:53:54.651471Z 2021-12-10T09:53:49.548712Z 2021-12-10T09:53:53.998404Z 2021-12-10T09:53:51.497934Z 2021-12-10T09:53:56.478173Z 2021-12-10T09:53:50.465293Z 2021-12-10T09:53:54.428263Z 2021-12-10T09:53:50.220286Z 2021-12-10T09:53:53.735412Z 2021-12-10T09:53:47.602867Z 2021-12-10T09:53:59.085263Z 2021-12-10T09:53:47.867764Z 2021-12-10T09:53:55.395405Z 2021-12-10T09:53:55.267999Z 2021-12-10T09:53:50.754526Z 2021-12-10T09:53:50.338549Z 2021-12-10T09:53:54.731226Z 2021-12-10T09:53:46.491280Z 2021-12-10T09:53:49.792538Z 2021-12-10T09:53:52.057840Z 2021-12-10T09:53:57.272128Z 2021-12-10T09:53:45.493271Z 2021-12-10T09:53:50.502723Z 2021-12-10T09:53:41.789841Z 2021-12-10T09:53:47.901584Z 2021-12-10T09:53:39.649310Z 2021-12-10T09:53:49.494498Z 2021-12-10T09:53:46.705329Z 2021-12-10T09:53:51.091877Z 2021-12-10T09:53:48.207624Z 2021-12-10T09:53:49.452043Z 2021-12-10T09:53:50.477304Z 2021-12-10T09:53:51.653118Z 2021-12-10T09:53:48.978973Z 2021-12-10T09:53:51.513446Z 2021-12-10T09:53:48.207624Z 2021-12-10T09:53:53.079935Z 2021-12-10T09:53:42.296268Z 2021-12-10T09:53:46.975508Z 2021-12-10T09:53:41.263333Z 2021-12-10T09:53:41.342444Z 2021-12-10T09:53:36.316678Z 2021-12-10T09:53:45.889441Z 2021-12-10T09:53:43.769469Z 2021-12-10T09:53:47.229440Z 2021-12-10T09:53:39.420912Z 2021-12-10T09:53:40.468543Z 2021-12-10T09:53:29.889476Z 2021-12-10T09:53:37.477404Z 2021-12-10T09:53:28.632738Z 2021-12-10T09:53:21.290025Z 2021-12-10T09:53:24.154140Z 2021-12-10T09:53:29.180435Z 2021-12-10T09:53:28.524281Z 2021-12-10T09:53:33.641475Z 2021-12-10T09:53:23.120337Z 2021-12-10T09:53:26.024397Z 2021-12-10T09:53:20.278412Z 2021-12-10T09:53:35.806416Z 2021-12-10T09:53:11.777675Z 2021-12-10T09:53:27.478889Z 2021-12-10T09:53:15.940973Z 2021-12-10T09:53:32.666439Z 2021-12-10T09:53:25.014299Z 2021-12-10T09:53:31.896465Z 2021-12-10T09:53:16.669263Z 2021-12-10T09:53:21.904677Z 2021-12-10T09:53:20.161004Z 2021-12-10T09:53:26.857436Z 2021-12-10T09:53:21.229220Z 2021-12-10T09:53:17.728812Z 2021-12-10T09:53:12.443314Z 2021-12-10T09:53:23.618661Z 2021-12-10T09:53:02.713483Z 2021-12-10T09:53:08.426300Z 2021-12-10T09:53:02.479633Z 2021-12-10T09:53:05.794421Z 2021-12-10T09:53:03.711279Z 2021-12-10T09:53:15.178590Z 2021-12-10T09:52:56.774492Z 2021-12-10T09:53:08.157177Z 2021-12-10T09:52:53.128510Z 2021-12-10T09:53:03.970405Z 2021-12-10T09:52:46.518707Z 2021-12-10T09:52:57.468437Z 2021-12-10T09:52:41.549294Z 2021-12-10T09:53:01.961778Z 2021-12-10T09:52:28.485818Z 2021-12-10T09:52:31.722930Z 2021-12-10T09:52:26.698435Z 2021-12-10T09:52:29.618529Z 2021-12-10T09:52:26.475477Z 2021-12-10T09:52:26.894383Z 2021-12-10T09:58:09.903969Z 2021-12-10T09:58:10.891712Z 2021-12-10T09:58:10.602385Z 2021-12-10T09:58:06.708411Z 2021-12-10T09:58:07.127289Z 2021-12-10T09:58:10.836299Z 2021-12-10T09:58:07.568437Z 2021-12-10T09:58:10.836299Z 2021-12-10T09:58:11.307171Z 2021-12-10T09:58:11.073248Z 2021-12-10T09:58:11.594551Z 2021-12-10T09:58:08.488353Z 2021-12-10T09:58:11.620046Z 2021-12-10T09:58:08.970155Z 2021-12-10T09:58:08.104809Z 2021-12-10T09:58:09.064868Z 2021-12-10T09:58:07.355427Z 2021-12-10T09:58:09.309791Z 2021-12-10T09:58:10.336584Z 2021-12-10T09:58:07.744468Z 2021-12-10T09:58:09.952655Z 2021-12-10T09:58:09.337038Z 2021-12-10T09:58:09.768460Z 2021-12-10T09:58:05.962699Z 2021-12-10T09:58:07.812140Z 2021-12-10T09:58:05.436552Z 2021-12-10T09:58:01.351469Z 2021-12-10T09:58:06.066275Z 2021-12-10T09:58:03.940354Z 2021-12-10T09:58:04.919475Z 2021-12-10T09:58:10.415999Z 2021-12-10T09:58:10.235154Z 2021-12-10T09:58:03.184462Z 2021-12-10T09:58:07.515449Z 2021-12-10T09:58:05.910414Z 2021-12-10T09:58:07.657463Z 2021-12-10T09:58:10.336584Z 2021-12-10T09:58:04.148321Z 2021-12-10T09:58:02.117431Z 2021-12-10T09:58:03.685502Z 2021-12-10T09:57:53.607463Z 2021-12-10T09:58:04.790399Z 2021-12-10T09:57:53.130296Z 2021-12-10T09:58:00.129502Z 2021-12-10T09:58:00.615289Z 2021-12-10T09:57:48.174590Z 2021-12-10T09:58:00.084563Z 2021-12-10T09:57:53.395098Z 2021-12-10T09:57:56.342653Z 2021-12-10T09:57:52.176437Z 2021-12-10T09:57:51.552822Z 2021-12-10T09:57:54.984598Z 2021-12-10T09:57:58.426511Z 2021-12-10T09:57:53.502474Z 2021-12-10T09:57:53.562293Z 2021-12-10T09:57:48.624513Z 2021-12-10T09:57:56.048809Z 2021-12-10T09:57:48.272519Z 2021-12-10T09:57:51.942587Z 2021-12-10T09:57:48.922425Z 2021-12-10T09:57:44.980315Z 2021-12-10T09:57:54.785282Z 2021-12-10T09:57:52.329231Z 2021-12-10T09:57:47.456482Z 2021-12-10T09:57:44.632307Z 2021-12-10T09:57:43.896276Z 2021-12-10T09:57:44.720356Z 2021-12-10T09:57:48.929399Z 2021-12-10T09:57:46.390562Z 2021-12-10T09:57:36.726286Z 2021-12-10T09:57:43.814616Z 2021-12-10T09:57:38.551410Z 2021-12-10T09:57:43.159641Z 2021-12-10T09:57:42.461230Z 2021-12-10T09:57:35.952382Z 2021-12-10T09:57:41.838868Z 2021-12-10T09:57:37.632561Z 2021-12-10T09:57:27.367464Z 2021-12-10T09:57:44.449300Z 2021-12-10T09:57:27.629414Z 2021-12-10T09:57:30.833993Z 2021-12-10T09:57:38.274016Z 2021-12-10T09:57:29.667283Z 2021-12-10T09:57:37.474545Z 2021-12-10T09:57:26.441404Z 2021-12-10T09:57:29.516559Z 2021-12-10T09:57:30.328880Z 2021-12-10T09:57:30.452280Z 2021-12-10T09:57:07.863255Z 2021-12-10T09:57:30.545690Z 2021-12-10T09:56:57.450651Z 2021-12-10T09:57:03.257455Z 2021-12-10T09:56:51.645262Z 2021-12-10T09:56:59.289404Z 2021-12-10T09:56:56.339727Z 2021-12-10T09:56:48.869483Z 2021-12-10T09:56:46.559004Z 2021-12-10T09:56:45.788426Z 2021-12-10T09:56:44.895339Z 2021-12-10T09:56:44.772790Z", + "failed_job_ids": " []", + "finished": "2021-12-10T09:40:28.478895Z 2021-12-10T09:39:45.494851Z 2021-12-10T09:40:22.289645Z 2021-12-10T09:39:45.829620Z 2021-12-10T09:40:27.746411Z 2021-12-10T09:39:36.882439Z 2021-12-10T09:40:28.885343Z 2021-12-10T09:39:36.393986Z 2021-12-10T09:40:26.756070Z 2021-12-10T09:39:37.528101Z 2021-12-10T09:40:04.565321Z 2021-12-10T09:41:03.142503Z 2021-12-10T09:40:01.635521Z 2021-12-10T09:40:54.423127Z 2021-12-10T09:40:08.643371Z 2021-12-10T09:41:50.465658Z 2021-12-10T09:40:02.360909Z 2021-12-10T09:40:36.744919Z 2021-12-10T09:40:04.658167Z 2021-12-10T09:41:41.370368Z 2021-12-10T09:40:07.776386Z 2021-12-10T09:41:12.563291Z 2021-12-10T09:39:57.687121Z 2021-12-10T09:40:45.968921Z 2021-12-10T09:41:31.884431Z 2021-12-10T09:39:55.137716Z 2021-12-10T09:39:30.629669Z 2021-12-10T09:41:22.240136Z 2021-12-10T09:39:29.628680Z 2021-12-10T09:40:23.620316Z 2021-12-10T09:39:28.947892Z 2021-12-10T09:42:00.566858Z 2021-12-10T09:39:17.270509Z 2021-12-10T09:39:30.461966Z 2021-12-10T09:39:10.484172Z 2021-12-10T09:40:31.561234Z 2021-12-10T09:40:42.949422Z 2021-12-10T09:40:29.926897Z 2021-12-10T09:40:43.526795Z 2021-12-10T09:40:31.218869Z 2021-12-10T09:40:31.901746Z 2021-12-10T09:40:32.138204Z 2021-12-10T09:40:28.646020Z 2021-12-10T09:40:31.559024Z 2021-12-10T09:40:43.160503Z 2021-12-10T09:40:29.433934Z 2021-12-10T09:40:41.678184Z 2021-12-10T09:40:27.972933Z 2021-12-10T09:40:40.690446Z 2021-12-10T09:40:28.133685Z 2021-12-10T09:40:41.431800Z 2021-12-10T09:40:27.645011Z 2021-12-10T09:40:40.862705Z 2021-12-10T09:40:28.308378Z 2021-12-10T09:40:42.726773Z 2021-12-10T09:40:24.986979Z 2021-12-10T09:40:42.517586Z 2021-12-10T09:40:24.824955Z 2021-12-10T09:40:40.729395Z 2021-12-10T09:40:23.664059Z 2021-12-10T09:40:42.065474Z 2021-12-10T09:40:22.178422Z 2021-12-10T09:40:40.933157Z 2021-12-10T09:40:21.617879Z 2021-12-10T09:40:41.858411Z 2021-12-10T09:40:18.279222Z 2021-12-10T09:40:35.651281Z 2021-12-10T09:40:28.743119Z 2021-12-10T09:40:41.739524Z 2021-12-10T09:40:26.480505Z 2021-12-10T09:40:36.140794Z 2021-12-10T09:40:25.640268Z 2021-12-10T09:40:38.652357Z 2021-12-10T09:40:20.368001Z 2021-12-10T09:40:39.231354Z 2021-12-10T09:40:16.101370Z 2021-12-10T09:40:37.019290Z 2021-12-10T09:39:50.602836Z 2021-12-10T09:40:39.121986Z 2021-12-10T09:40:38.538415Z 2021-12-10T09:40:29.081247Z 2021-12-10T09:40:00.735140Z 2021-12-10T09:40:30.094405Z 2021-12-10T09:39:52.400206Z 2021-12-10T09:40:39.265973Z 2021-12-10T09:39:56.099437Z 2021-12-10T09:40:22.462445Z 2021-12-10T09:39:48.716289Z 2021-12-10T09:40:32.139663Z 2021-12-10T09:39:53.947622Z 2021-12-10T09:40:29.742302Z 2021-12-10T09:39:52.061229Z 2021-12-10T09:40:24.176722Z 2021-12-10T09:39:50.595981Z 2021-12-10T09:40:22.354326Z 2021-12-10T09:39:48.917051Z 2021-12-10T09:40:27.043464Z 2021-12-10T09:39:43.486380Z 2021-12-10T09:40:22.548079Z 2021-12-10T09:39:51.390416Z 2021-12-10T09:45:35.405421Z 2021-12-10T09:45:38.220990Z 2021-12-10T09:45:34.961673Z 2021-12-10T09:45:34.112489Z 2021-12-10T09:45:35.282124Z 2021-12-10T09:45:36.189395Z 2021-12-10T09:45:33.326275Z 2021-12-10T09:45:37.034249Z 2021-12-10T09:45:34.155714Z 2021-12-10T09:45:37.629775Z 2021-12-10T09:45:32.759140Z 2021-12-10T09:45:37.037319Z 2021-12-10T09:45:34.152821Z 2021-12-10T09:45:33.610568Z 2021-12-10T09:45:34.202469Z 2021-12-10T09:45:37.443368Z 2021-12-10T09:45:33.627591Z 2021-12-10T09:45:35.672719Z 2021-12-10T09:45:34.306268Z 2021-12-10T09:45:33.430358Z 2021-12-10T09:45:34.111671Z 2021-12-10T09:45:35.558091Z 2021-12-10T09:45:34.015974Z 2021-12-10T09:45:36.711372Z 2021-12-10T09:45:37.337755Z 2021-12-10T09:45:24.903957Z 2021-12-10T09:45:30.578218Z 2021-12-10T09:45:29.303274Z 2021-12-10T09:45:30.476235Z 2021-12-10T09:45:29.317740Z 2021-12-10T09:45:31.758897Z 2021-12-10T09:45:26.732209Z 2021-12-10T09:45:26.510139Z 2021-12-10T09:45:28.303667Z 2021-12-10T09:45:30.495870Z 2021-12-10T09:45:31.629814Z 2021-12-10T09:45:37.797242Z 2021-12-10T09:45:26.755228Z 2021-12-10T09:45:29.893143Z 2021-12-10T09:45:29.548387Z 2021-12-10T09:45:25.211351Z 2021-12-10T09:45:28.590772Z 2021-12-10T09:45:32.522167Z 2021-12-10T09:45:19.145000Z 2021-12-10T09:45:30.159175Z 2021-12-10T09:45:17.423435Z 2021-12-10T09:45:22.474128Z 2021-12-10T09:45:30.805957Z 2021-12-10T09:45:18.198134Z 2021-12-10T09:45:30.275500Z 2021-12-10T09:45:18.421043Z 2021-12-10T09:45:24.573246Z 2021-12-10T09:45:19.339503Z 2021-12-10T09:45:22.064171Z 2021-12-10T09:45:25.959493Z 2021-12-10T09:45:24.872275Z 2021-12-10T09:45:18.652437Z 2021-12-10T09:45:24.283241Z 2021-12-10T09:45:20.922202Z 2021-12-10T09:45:18.696499Z 2021-12-10T09:45:14.855346Z 2021-12-10T09:45:17.634096Z 2021-12-10T09:45:19.237047Z 2021-12-10T09:45:15.360560Z 2021-12-10T09:45:12.680956Z 2021-12-10T09:45:01.956129Z 2021-12-10T09:45:19.651539Z 2021-12-10T09:45:10.378422Z 2021-12-10T09:45:06.979546Z 2021-12-10T09:45:13.077756Z 2021-12-10T09:45:04.653291Z 2021-12-10T09:45:05.373565Z 2021-12-10T09:45:14.115085Z 2021-12-10T09:45:06.859237Z 2021-12-10T09:45:01.403209Z 2021-12-10T09:45:00.038246Z 2021-12-10T09:45:04.894766Z 2021-12-10T09:44:53.957606Z 2021-12-10T09:44:34.202758Z 2021-12-10T09:44:27.453332Z 2021-12-10T09:44:37.778569Z 2021-12-10T09:44:27.405992Z 2021-12-10T09:44:28.693104Z 2021-12-10T09:44:29.741834Z 2021-12-10T09:44:31.329608Z 2021-12-10T09:44:25.748353Z 2021-12-10T09:44:22.386403Z 2021-12-10T09:44:22.696722Z 2021-12-10T09:44:26.881759Z 2021-12-10T09:44:22.709900Z 2021-12-10T09:44:23.011431Z 2021-12-10T09:44:21.637243Z 2021-12-10T09:44:18.774965Z 2021-12-10T09:44:20.961870Z 2021-12-10T09:44:20.076496Z 2021-12-10T09:44:15.227725Z 2021-12-10T09:44:10.685374Z 2021-12-10T09:44:10.523950Z 2021-12-10T09:44:07.266987Z 2021-12-10T09:44:04.730072Z 2021-12-10T09:49:50.101237Z 2021-12-10T09:49:50.139723Z 2021-12-10T09:49:51.518792Z 2021-12-10T09:49:48.607929Z 2021-12-10T09:49:51.253231Z 2021-12-10T09:49:48.118858Z 2021-12-10T09:49:51.683091Z 2021-12-10T09:49:49.251178Z 2021-12-10T09:49:52.020577Z 2021-12-10T09:49:46.453109Z 2021-12-10T09:49:49.447881Z 2021-12-10T09:49:49.890254Z 2021-12-10T09:49:51.787500Z 2021-12-10T09:49:43.973334Z 2021-12-10T09:49:52.655917Z 2021-12-10T09:49:44.913663Z 2021-12-10T09:49:46.955865Z 2021-12-10T09:49:39.537096Z 2021-12-10T09:49:49.669237Z 2021-12-10T09:49:49.447281Z 2021-12-10T09:49:46.346441Z 2021-12-10T09:49:48.664759Z 2021-12-10T09:49:49.801081Z 2021-12-10T09:49:48.854682Z 2021-12-10T09:49:46.837142Z 2021-12-10T09:49:49.541907Z 2021-12-10T09:49:51.411973Z 2021-12-10T09:49:47.498694Z 2021-12-10T09:49:45.143057Z 2021-12-10T09:49:46.702680Z 2021-12-10T09:49:50.615078Z 2021-12-10T09:49:45.573445Z 2021-12-10T09:49:47.998369Z 2021-12-10T09:49:47.687630Z 2021-12-10T09:49:46.027573Z 2021-12-10T09:49:43.800297Z 2021-12-10T09:49:45.902424Z 2021-12-10T09:49:45.039961Z 2021-12-10T09:49:47.563363Z 2021-12-10T09:49:42.145809Z 2021-12-10T09:49:39.232232Z 2021-12-10T09:49:41.113255Z 2021-12-10T09:49:44.211305Z 2021-12-10T09:48:55.432737Z 2021-12-10T09:49:43.530565Z 2021-12-10T09:49:45.040212Z 2021-12-10T09:49:44.344259Z 2021-12-10T09:49:38.418171Z 2021-12-10T09:49:39.311347Z 2021-12-10T09:49:44.937262Z 2021-12-10T09:49:36.126496Z 2021-12-10T09:49:46.458974Z 2021-12-10T09:49:42.261569Z 2021-12-10T09:49:40.209188Z 2021-12-10T09:49:35.244166Z 2021-12-10T09:49:33.312632Z 2021-12-10T09:49:36.435597Z 2021-12-10T09:49:40.202922Z 2021-12-10T09:49:39.303147Z 2021-12-10T09:49:27.504737Z 2021-12-10T09:49:34.798175Z 2021-12-10T09:49:20.088446Z 2021-12-10T09:49:38.298391Z 2021-12-10T09:49:31.852375Z 2021-12-10T09:49:27.220967Z 2021-12-10T09:49:25.035395Z 2021-12-10T09:49:24.098057Z 2021-12-10T09:49:19.129425Z 2021-12-10T09:49:22.411126Z 2021-12-10T09:49:16.214273Z 2021-12-10T09:49:15.894036Z 2021-12-10T09:49:25.570482Z 2021-12-10T09:49:16.647220Z 2021-12-10T09:49:12.811326Z 2021-12-10T09:49:15.859243Z 2021-12-10T09:49:17.491218Z 2021-12-10T09:49:11.252256Z 2021-12-10T09:49:08.578308Z 2021-12-10T09:49:06.144799Z 2021-12-10T09:49:14.535429Z 2021-12-10T09:49:13.075436Z 2021-12-10T09:49:16.189413Z 2021-12-10T09:49:05.945246Z 2021-12-10T09:49:02.206701Z 2021-12-10T09:49:01.995053Z 2021-12-10T09:48:53.237145Z 2021-12-10T09:48:56.335260Z 2021-12-10T09:49:05.303435Z 2021-12-10T09:48:48.353713Z 2021-12-10T09:48:58.150541Z 2021-12-10T09:48:47.753592Z 2021-12-10T09:48:52.714357Z 2021-12-10T09:48:42.254615Z 2021-12-10T09:48:45.495433Z 2021-12-10T09:48:26.454727Z 2021-12-10T09:48:28.963491Z 2021-12-10T09:48:24.324201Z 2021-12-10T09:48:24.273365Z 2021-12-10T09:48:22.181691Z 2021-12-10T09:48:19.070832Z 2021-12-10T09:53:55.766183Z 2021-12-10T09:54:00.428498Z 2021-12-10T09:53:56.776532Z 2021-12-10T09:54:01.324495Z 2021-12-10T09:53:57.329723Z 2021-12-10T09:54:00.708540Z 2021-12-10T09:53:55.254718Z 2021-12-10T09:53:58.899651Z 2021-12-10T09:53:53.963081Z 2021-12-10T09:53:59.252518Z 2021-12-10T09:53:56.699912Z 2021-12-10T09:54:00.648863Z 2021-12-10T09:53:54.852561Z 2021-12-10T09:53:58.936481Z 2021-12-10T09:53:54.442059Z 2021-12-10T09:53:59.280125Z 2021-12-10T09:53:53.725988Z 2021-12-10T09:54:01.948111Z 2021-12-10T09:53:51.732664Z 2021-12-10T09:53:59.722960Z 2021-12-10T09:53:59.306475Z 2021-12-10T09:53:55.546841Z 2021-12-10T09:53:55.369933Z 2021-12-10T09:53:59.282574Z 2021-12-10T09:53:51.809671Z 2021-12-10T09:53:55.960475Z 2021-12-10T09:53:56.397426Z 2021-12-10T09:54:01.035906Z 2021-12-10T09:53:51.156353Z 2021-12-10T09:53:54.854559Z 2021-12-10T09:53:46.385960Z 2021-12-10T09:53:53.037139Z 2021-12-10T09:53:44.493481Z 2021-12-10T09:53:54.865032Z 2021-12-10T09:53:52.290972Z 2021-12-10T09:53:56.527541Z 2021-12-10T09:53:52.144976Z 2021-12-10T09:53:53.515726Z 2021-12-10T09:53:55.999682Z 2021-12-10T09:53:55.672687Z 2021-12-10T09:53:53.638879Z 2021-12-10T09:53:57.285256Z 2021-12-10T09:53:53.635769Z 2021-12-10T09:53:57.421680Z 2021-12-10T09:53:47.007717Z 2021-12-10T09:53:52.059419Z 2021-12-10T09:53:46.524404Z 2021-12-10T09:53:46.325557Z 2021-12-10T09:53:41.468997Z 2021-12-10T09:53:51.238727Z 2021-12-10T09:53:48.277115Z 2021-12-10T09:53:52.332113Z 2021-12-10T09:53:44.547265Z 2021-12-10T09:53:45.747141Z 2021-12-10T09:53:34.715249Z 2021-12-10T09:53:42.377273Z 2021-12-10T09:53:33.117212Z 2021-12-10T09:53:27.450972Z 2021-12-10T09:53:29.275150Z 2021-12-10T09:53:34.507895Z 2021-12-10T09:53:34.557993Z 2021-12-10T09:53:39.305343Z 2021-12-10T09:53:29.279822Z 2021-12-10T09:53:31.073132Z 2021-12-10T09:53:26.935144Z 2021-12-10T09:53:40.239486Z 2021-12-10T09:53:17.887243Z 2021-12-10T09:53:34.212211Z 2021-12-10T09:53:20.008139Z 2021-12-10T09:53:38.884080Z 2021-12-10T09:53:31.138636Z 2021-12-10T09:53:37.891867Z 2021-12-10T09:53:22.771338Z 2021-12-10T09:53:26.335300Z 2021-12-10T09:53:25.980390Z 2021-12-10T09:53:32.084489Z 2021-12-10T09:53:27.710088Z 2021-12-10T09:53:23.235972Z 2021-12-10T09:53:19.147826Z 2021-12-10T09:53:30.050402Z 2021-12-10T09:53:10.259509Z 2021-12-10T09:53:15.173270Z 2021-12-10T09:53:10.464373Z 2021-12-10T09:53:12.771158Z 2021-12-10T09:53:09.733670Z 2021-12-10T09:53:20.675297Z 2021-12-10T09:53:03.288713Z 2021-12-10T09:53:12.865360Z 2021-12-10T09:52:58.655199Z 2021-12-10T09:53:09.819167Z 2021-12-10T09:52:51.732195Z 2021-12-10T09:53:02.281248Z 2021-12-10T09:52:46.232960Z 2021-12-10T09:53:08.302280Z 2021-12-10T09:52:32.731031Z 2021-12-10T09:52:36.703492Z 2021-12-10T09:52:30.264467Z 2021-12-10T09:52:31.203495Z 2021-12-10T09:52:30.808835Z 2021-12-10T09:52:30.560309Z 2021-12-10T09:58:14.338174Z 2021-12-10T09:58:15.149061Z 2021-12-10T09:58:14.790359Z 2021-12-10T09:58:11.360541Z 2021-12-10T09:58:11.845814Z 2021-12-10T09:58:14.603286Z 2021-12-10T09:58:12.199931Z 2021-12-10T09:58:15.327627Z 2021-12-10T09:58:15.214346Z 2021-12-10T09:58:15.288134Z 2021-12-10T09:58:16.002165Z 2021-12-10T09:58:13.280842Z 2021-12-10T09:58:15.217556Z 2021-12-10T09:58:13.300920Z 2021-12-10T09:58:12.832961Z 2021-12-10T09:58:13.055393Z 2021-12-10T09:58:13.162373Z 2021-12-10T09:58:14.193132Z 2021-12-10T09:58:15.225977Z 2021-12-10T09:58:13.227733Z 2021-12-10T09:58:14.335774Z 2021-12-10T09:58:14.071919Z 2021-12-10T09:58:15.067694Z 2021-12-10T09:58:11.420043Z 2021-12-10T09:58:11.971651Z 2021-12-10T09:58:10.617658Z 2021-12-10T09:58:05.709403Z 2021-12-10T09:58:11.069630Z 2021-12-10T09:58:08.846813Z 2021-12-10T09:58:09.131565Z 2021-12-10T09:58:15.172474Z 2021-12-10T09:58:14.967101Z 2021-12-10T09:58:08.144739Z 2021-12-10T09:58:11.959754Z 2021-12-10T09:58:10.640476Z 2021-12-10T09:58:12.759032Z 2021-12-10T09:58:14.850662Z 2021-12-10T09:58:08.261772Z 2021-12-10T09:58:07.370413Z 2021-12-10T09:58:09.134449Z 2021-12-10T09:58:00.490337Z 2021-12-10T09:58:09.603685Z 2021-12-10T09:57:58.323039Z 2021-12-10T09:58:05.462557Z 2021-12-10T09:58:05.649381Z 2021-12-10T09:57:55.775288Z 2021-12-10T09:58:04.334128Z 2021-12-10T09:57:58.726355Z 2021-12-10T09:58:01.463188Z 2021-12-10T09:57:57.685357Z 2021-12-10T09:57:56.137388Z 2021-12-10T09:58:00.235258Z 2021-12-10T09:58:04.411393Z 2021-12-10T09:57:59.750189Z 2021-12-10T09:57:58.497479Z 2021-12-10T09:57:54.814856Z 2021-12-10T09:58:00.251039Z 2021-12-10T09:57:55.598553Z 2021-12-10T09:57:57.586206Z 2021-12-10T09:57:55.017017Z 2021-12-10T09:57:51.256594Z 2021-12-10T09:57:59.743178Z 2021-12-10T09:57:57.842006Z 2021-12-10T09:57:53.825765Z 2021-12-10T09:57:49.665158Z 2021-12-10T09:57:49.980746Z 2021-12-10T09:57:50.593283Z 2021-12-10T09:57:54.151487Z 2021-12-10T09:57:51.760038Z 2021-12-10T09:57:41.465137Z 2021-12-10T09:57:49.367257Z 2021-12-10T09:57:44.355221Z 2021-12-10T09:57:49.789409Z 2021-12-10T09:57:48.468309Z 2021-12-10T09:57:42.765191Z 2021-12-10T09:57:47.499284Z 2021-12-10T09:57:42.601706Z 2021-12-10T09:57:34.045274Z 2021-12-10T09:57:49.663182Z 2021-12-10T09:57:34.448413Z 2021-12-10T09:57:36.876187Z 2021-12-10T09:57:43.835303Z 2021-12-10T09:57:35.694257Z 2021-12-10T09:57:42.685556Z 2021-12-10T09:57:30.750919Z 2021-12-10T09:57:34.572090Z 2021-12-10T09:57:35.343410Z 2021-12-10T09:57:36.133807Z 2021-12-10T09:57:12.872684Z 2021-12-10T09:57:35.364235Z 2021-12-10T09:57:02.781156Z 2021-12-10T09:57:08.208270Z 2021-12-10T09:56:56.420021Z 2021-12-10T09:57:04.497389Z 2021-12-10T09:57:01.813141Z 2021-12-10T09:56:53.318034Z 2021-12-10T09:56:50.765155Z 2021-12-10T09:56:49.817460Z 2021-12-10T09:56:48.376892Z 2021-12-10T09:56:48.861992Z", + "finished_diff": 107.29352120000001, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 180.314978, + "mean": 54.08233803006008, + "min": 11.88277 + }, + "jobs_events_duration": { + "max": 8.125825, + "mean": 3.121091521042085, + "min": 0.164075 + }, + "jobs_events_lag": { + "max": 339.34366, + "mean": -4.066341569138276, + "min": -39.816462 + }, + "jobs_waiting": { + "max": 39.966939, + "mean": 21.9332598476954, + "min": 0.567066 + }, + "modified": "2021-12-10T09:39:10.187974Z 2021-12-10T09:39:10.113411Z 2021-12-10T09:39:09.903168Z 2021-12-10T09:39:09.750238Z 2021-12-10T09:39:09.650981Z 2021-12-10T09:39:09.528887Z 2021-12-10T09:39:09.435218Z 2021-12-10T09:39:09.367233Z 2021-12-10T09:39:09.266687Z 2021-12-10T09:39:09.148895Z 2021-12-10T09:39:03.948675Z 2021-12-10T09:39:03.879344Z 2021-12-10T09:39:03.826789Z 2021-12-10T09:39:03.794368Z 2021-12-10T09:39:03.736962Z 2021-12-10T09:39:03.683919Z 2021-12-10T09:39:03.628798Z 2021-12-10T09:39:03.516945Z 2021-12-10T09:39:03.458234Z 2021-12-10T09:39:03.361607Z 2021-12-10T09:39:03.266811Z 2021-12-10T09:39:03.212222Z 2021-12-10T09:39:03.143263Z 2021-12-10T09:39:03.078217Z 2021-12-10T09:39:01.034119Z 2021-12-10T09:39:03.003583Z 2021-12-10T09:39:00.918333Z 2021-12-10T09:39:00.775598Z 2021-12-10T09:39:00.664689Z 2021-12-10T09:39:00.588230Z 2021-12-10T09:39:00.522648Z 2021-12-10T09:38:59.865156Z 2021-12-10T09:38:59.822629Z 2021-12-10T09:38:59.388497Z 2021-12-10T09:38:58.440500Z 2021-12-10T09:39:50.555988Z 2021-12-10T09:39:50.462677Z 2021-12-10T09:39:50.374467Z 2021-12-10T09:39:50.286020Z 2021-12-10T09:39:50.170971Z 2021-12-10T09:39:50.032550Z 2021-12-10T09:39:49.839915Z 2021-12-10T09:39:32.882506Z 2021-12-10T09:39:49.671371Z 2021-12-10T09:39:32.800188Z 2021-12-10T09:39:32.703006Z 2021-12-10T09:39:32.621438Z 2021-12-10T09:39:32.547361Z 2021-12-10T09:39:32.480584Z 2021-12-10T09:39:32.393812Z 2021-12-10T09:39:32.313544Z 2021-12-10T09:39:32.239064Z 2021-12-10T09:39:32.135465Z 2021-12-10T09:39:32.049107Z 2021-12-10T09:39:31.967381Z 2021-12-10T09:39:31.886444Z 2021-12-10T09:39:31.792913Z 2021-12-10T09:39:31.696765Z 2021-12-10T09:39:31.582660Z 2021-12-10T09:39:31.478328Z 2021-12-10T09:39:31.313726Z 2021-12-10T09:39:31.197128Z 2021-12-10T09:39:31.121553Z 2021-12-10T09:39:31.004136Z 2021-12-10T09:39:30.892939Z 2021-12-10T09:39:30.796499Z 2021-12-10T09:39:30.728139Z 2021-12-10T09:39:30.633596Z 2021-12-10T09:39:30.464231Z 2021-12-10T09:39:30.342382Z 2021-12-10T09:39:30.193038Z 2021-12-10T09:39:30.070177Z 2021-12-10T09:39:29.956845Z 2021-12-10T09:39:29.808951Z 2021-12-10T09:39:29.535019Z 2021-12-10T09:39:29.364652Z 2021-12-10T09:39:29.213176Z 2021-12-10T09:39:11.620728Z 2021-12-10T09:39:28.934185Z 2021-12-10T09:39:28.742593Z 2021-12-10T09:39:11.553015Z 2021-12-10T09:39:11.514267Z 2021-12-10T09:39:11.470230Z 2021-12-10T09:39:11.432900Z 2021-12-10T09:39:11.393633Z 2021-12-10T09:39:11.346628Z 2021-12-10T09:39:11.304481Z 2021-12-10T09:39:11.263198Z 2021-12-10T09:39:11.212406Z 2021-12-10T09:39:11.130523Z 2021-12-10T09:39:11.049050Z 2021-12-10T09:39:10.965311Z 2021-12-10T09:39:10.878029Z 2021-12-10T09:39:10.779298Z 2021-12-10T09:39:10.672522Z 2021-12-10T09:39:10.606234Z 2021-12-10T09:39:10.551244Z 2021-12-10T09:39:10.501995Z 2021-12-10T09:39:10.397333Z 2021-12-10T09:39:10.240378Z 2021-12-10T09:44:37.034166Z 2021-12-10T09:44:36.984115Z 2021-12-10T09:44:36.937194Z 2021-12-10T09:44:36.886265Z 2021-12-10T09:44:36.838375Z 2021-12-10T09:44:36.783270Z 2021-12-10T09:44:36.733929Z 2021-12-10T09:44:36.691424Z 2021-12-10T09:44:36.650348Z 2021-12-10T09:44:36.604286Z 2021-12-10T09:44:36.561823Z 2021-12-10T09:44:36.515332Z 2021-12-10T09:44:36.467439Z 2021-12-10T09:44:36.426119Z 2021-12-10T09:44:36.379601Z 2021-12-10T09:44:36.318214Z 2021-12-10T09:44:36.265385Z 2021-12-10T09:44:36.210705Z 2021-12-10T09:44:36.159575Z 2021-12-10T09:44:36.093268Z 2021-12-10T09:44:36.038864Z 2021-12-10T09:44:35.971143Z 2021-12-10T09:44:35.908416Z 2021-12-10T09:44:35.859929Z 2021-12-10T09:44:35.815270Z 2021-12-10T09:44:15.913365Z 2021-12-10T09:44:15.790118Z 2021-12-10T09:44:15.663555Z 2021-12-10T09:44:15.582883Z 2021-12-10T09:44:15.478345Z 2021-12-10T09:44:15.387190Z 2021-12-10T09:44:15.286660Z 2021-12-10T09:44:15.197176Z 2021-12-10T09:44:15.128057Z 2021-12-10T09:44:15.054365Z 2021-12-10T09:44:14.957472Z 2021-12-10T09:44:35.741160Z 2021-12-10T09:44:14.848497Z 2021-12-10T09:44:14.700380Z 2021-12-10T09:44:14.554559Z 2021-12-10T09:44:14.445232Z 2021-12-10T09:44:14.316641Z 2021-12-10T09:44:14.250489Z 2021-12-10T09:44:14.176362Z 2021-12-10T09:44:14.111302Z 2021-12-10T09:44:14.039321Z 2021-12-10T09:44:13.931453Z 2021-12-10T09:44:13.806212Z 2021-12-10T09:44:13.698155Z 2021-12-10T09:44:13.576797Z 2021-12-10T09:44:13.534919Z 2021-12-10T09:44:13.499778Z 2021-12-10T09:44:13.319366Z 2021-12-10T09:44:13.136336Z 2021-12-10T09:44:12.966328Z 2021-12-10T09:44:12.814345Z 2021-12-10T09:44:12.654364Z 2021-12-10T09:44:12.445903Z 2021-12-10T09:44:12.275290Z 2021-12-10T09:44:12.120376Z 2021-12-10T09:44:11.934392Z 2021-12-10T09:44:11.792227Z 2021-12-10T09:44:11.663196Z 2021-12-10T09:44:11.599117Z 2021-12-10T09:44:11.546853Z 2021-12-10T09:44:11.425505Z 2021-12-10T09:44:11.323194Z 2021-12-10T09:44:11.231535Z 2021-12-10T09:44:11.143314Z 2021-12-10T09:44:11.018959Z 2021-12-10T09:44:10.928713Z 2021-12-10T09:44:10.785465Z 2021-12-10T09:44:10.674716Z 2021-12-10T09:44:10.446680Z 2021-12-10T09:44:10.233399Z 2021-12-10T09:44:09.960486Z 2021-12-10T09:44:09.637865Z 2021-12-10T09:44:09.369054Z 2021-12-10T09:43:57.165369Z 2021-12-10T09:43:57.068980Z 2021-12-10T09:43:56.992635Z 2021-12-10T09:43:56.852275Z 2021-12-10T09:43:56.787330Z 2021-12-10T09:43:56.677775Z 2021-12-10T09:43:56.592276Z 2021-12-10T09:43:56.526663Z 2021-12-10T09:43:56.453375Z 2021-12-10T09:43:56.378722Z 2021-12-10T09:43:56.283101Z 2021-12-10T09:43:56.136652Z 2021-12-10T09:43:55.979335Z 2021-12-10T09:43:55.911151Z 2021-12-10T09:43:55.847903Z 2021-12-10T09:43:55.771900Z 2021-12-10T09:43:55.641701Z 2021-12-10T09:43:53.732061Z 2021-12-10T09:43:53.617933Z 2021-12-10T09:43:53.523266Z 2021-12-10T09:43:53.441551Z 2021-12-10T09:43:52.465108Z 2021-12-10T09:48:33.437208Z 2021-12-10T09:48:33.379240Z 2021-12-10T09:48:33.324377Z 2021-12-10T09:48:33.272158Z 2021-12-10T09:48:33.213260Z 2021-12-10T09:48:33.155311Z 2021-12-10T09:48:33.098420Z 2021-12-10T09:48:33.022119Z 2021-12-10T09:48:32.940192Z 2021-12-10T09:48:32.849350Z 2021-12-10T09:48:32.776372Z 2021-12-10T09:48:32.704373Z 2021-12-10T09:48:32.621372Z 2021-12-10T09:48:32.496216Z 2021-12-10T09:48:32.375201Z 2021-12-10T09:48:32.296822Z 2021-12-10T09:48:32.257175Z 2021-12-10T09:48:32.215938Z 2021-12-10T09:48:32.181809Z 2021-12-10T09:48:32.141364Z 2021-12-10T09:48:32.100129Z 2021-12-10T09:48:32.057420Z 2021-12-10T09:48:32.008250Z 2021-12-10T09:48:31.958193Z 2021-12-10T09:48:31.908166Z 2021-12-10T09:48:31.862350Z 2021-12-10T09:48:31.819721Z 2021-12-10T09:48:31.779484Z 2021-12-10T09:48:31.731141Z 2021-12-10T09:48:31.690590Z 2021-12-10T09:48:31.644145Z 2021-12-10T09:48:31.601190Z 2021-12-10T09:48:31.563434Z 2021-12-10T09:48:31.525815Z 2021-12-10T09:48:31.481831Z 2021-12-10T09:48:31.435411Z 2021-12-10T09:48:31.361375Z 2021-12-10T09:48:31.292367Z 2021-12-10T09:48:31.223305Z 2021-12-10T09:48:31.127293Z 2021-12-10T09:48:30.999173Z 2021-12-10T09:48:30.888053Z 2021-12-10T09:48:30.788431Z 2021-12-10T09:48:30.690737Z 2021-12-10T09:48:30.583171Z 2021-12-10T09:48:30.473263Z 2021-12-10T09:48:30.419227Z 2021-12-10T09:48:30.346272Z 2021-12-10T09:48:30.259278Z 2021-12-10T09:48:30.176301Z 2021-12-10T09:48:30.093217Z 2021-12-10T09:48:30.019266Z 2021-12-10T09:48:29.949352Z 2021-12-10T09:48:29.897121Z 2021-12-10T09:48:29.848378Z 2021-12-10T09:48:29.802654Z 2021-12-10T09:48:29.751332Z 2021-12-10T09:48:29.696540Z 2021-12-10T09:48:29.640476Z 2021-12-10T09:48:18.988085Z 2021-12-10T09:48:29.586704Z 2021-12-10T09:48:18.949535Z 2021-12-10T09:48:29.541454Z 2021-12-10T09:48:18.911241Z 2021-12-10T09:48:18.790413Z 2021-12-10T09:48:18.705894Z 2021-12-10T09:48:18.627145Z 2021-12-10T09:48:18.523430Z 2021-12-10T09:48:18.443413Z 2021-12-10T09:48:18.249505Z 2021-12-10T09:48:18.151684Z 2021-12-10T09:48:18.005597Z 2021-12-10T09:48:17.882698Z 2021-12-10T09:48:17.749816Z 2021-12-10T09:48:17.589935Z 2021-12-10T09:48:17.429284Z 2021-12-10T09:48:17.280487Z 2021-12-10T09:48:17.134790Z 2021-12-10T09:48:16.985359Z 2021-12-10T09:48:16.848851Z 2021-12-10T09:48:16.636529Z 2021-12-10T09:48:16.515059Z 2021-12-10T09:48:16.067436Z 2021-12-10T09:48:15.770123Z 2021-12-10T09:48:15.305378Z 2021-12-10T09:48:14.920458Z 2021-12-10T09:48:14.650170Z 2021-12-10T09:48:14.429425Z 2021-12-10T09:48:14.245893Z 2021-12-10T09:48:13.765847Z 2021-12-10T09:48:13.607964Z 2021-12-10T09:48:13.412510Z 2021-12-10T09:48:13.291729Z 2021-12-10T09:48:13.180919Z 2021-12-10T09:48:08.910060Z 2021-12-10T09:48:08.819576Z 2021-12-10T09:48:08.703413Z 2021-12-10T09:48:08.550990Z 2021-12-10T09:48:08.398851Z 2021-12-10T09:48:06.856768Z 2021-12-10T09:52:45.383278Z 2021-12-10T09:52:45.332504Z 2021-12-10T09:52:45.285247Z 2021-12-10T09:52:45.232365Z 2021-12-10T09:52:45.155570Z 2021-12-10T09:52:45.078333Z 2021-12-10T09:52:44.995467Z 2021-12-10T09:52:44.911385Z 2021-12-10T09:52:44.837515Z 2021-12-10T09:52:44.765662Z 2021-12-10T09:52:44.675500Z 2021-12-10T09:52:44.600405Z 2021-12-10T09:52:44.513571Z 2021-12-10T09:52:44.422877Z 2021-12-10T09:52:44.339446Z 2021-12-10T09:52:44.271252Z 2021-12-10T09:52:44.191840Z 2021-12-10T09:52:44.107947Z 2021-12-10T09:52:44.046992Z 2021-12-10T09:52:43.995491Z 2021-12-10T09:52:43.707636Z 2021-12-10T09:52:43.817461Z 2021-12-10T09:52:43.603429Z 2021-12-10T09:52:43.526983Z 2021-12-10T09:52:43.444500Z 2021-12-10T09:52:43.364356Z 2021-12-10T09:52:43.288794Z 2021-12-10T09:52:43.210808Z 2021-12-10T09:52:43.142812Z 2021-12-10T09:52:43.073483Z 2021-12-10T09:52:43.002317Z 2021-12-10T09:52:42.927659Z 2021-12-10T09:52:42.844467Z 2021-12-10T09:52:42.755414Z 2021-12-10T09:52:42.665314Z 2021-12-10T09:52:42.556997Z 2021-12-10T09:52:42.469643Z 2021-12-10T09:52:42.361488Z 2021-12-10T09:52:42.157592Z 2021-12-10T09:52:42.264438Z 2021-12-10T09:52:42.047287Z 2021-12-10T09:52:41.961418Z 2021-12-10T09:52:41.875703Z 2021-12-10T09:52:41.787301Z 2021-12-10T09:52:41.707344Z 2021-12-10T09:52:41.615444Z 2021-12-10T09:52:41.522438Z 2021-12-10T09:52:41.441301Z 2021-12-10T09:52:41.339355Z 2021-12-10T09:52:41.207571Z 2021-12-10T09:52:41.091468Z 2021-12-10T09:52:40.999442Z 2021-12-10T09:52:26.559039Z 2021-12-10T09:52:40.884217Z 2021-12-10T09:52:26.473000Z 2021-12-10T09:52:26.399286Z 2021-12-10T09:52:26.337680Z 2021-12-10T09:52:26.230536Z 2021-12-10T09:52:26.146395Z 2021-12-10T09:52:26.094626Z 2021-12-10T09:52:26.012999Z 2021-12-10T09:52:25.933308Z 2021-12-10T09:52:25.819726Z 2021-12-10T09:52:25.765415Z 2021-12-10T09:52:25.655918Z 2021-12-10T09:52:25.615873Z 2021-12-10T09:52:25.544623Z 2021-12-10T09:52:25.479362Z 2021-12-10T09:52:25.444440Z 2021-12-10T09:52:25.401323Z 2021-12-10T09:52:25.282184Z 2021-12-10T09:52:25.213013Z 2021-12-10T09:52:25.054561Z 2021-12-10T09:52:24.938516Z 2021-12-10T09:52:24.868867Z 2021-12-10T09:52:24.826729Z 2021-12-10T09:52:24.714493Z 2021-12-10T09:52:24.789927Z 2021-12-10T09:52:24.583519Z 2021-12-10T09:52:24.414731Z 2021-12-10T09:52:24.269728Z 2021-12-10T09:52:24.128426Z 2021-12-10T09:52:23.984688Z 2021-12-10T09:52:23.801616Z 2021-12-10T09:52:23.689556Z 2021-12-10T09:52:23.505270Z 2021-12-10T09:52:23.234962Z 2021-12-10T09:52:23.007525Z 2021-12-10T09:52:22.742816Z 2021-12-10T09:52:22.566269Z 2021-12-10T09:52:22.427170Z 2021-12-10T09:52:22.313751Z 2021-12-10T09:52:22.199746Z 2021-12-10T09:52:22.089701Z 2021-12-10T09:52:16.201225Z 2021-12-10T09:52:16.067006Z 2021-12-10T09:52:15.882622Z 2021-12-10T09:52:15.769377Z 2021-12-10T09:52:15.684189Z 2021-12-10T09:52:15.605890Z 2021-12-10T09:57:06.103314Z 2021-12-10T09:57:06.000296Z 2021-12-10T09:57:05.902866Z 2021-12-10T09:57:05.781002Z 2021-12-10T09:57:05.843208Z 2021-12-10T09:57:05.723341Z 2021-12-10T09:57:05.664254Z 2021-12-10T09:57:05.613942Z 2021-12-10T09:57:05.564133Z 2021-12-10T09:57:05.502322Z 2021-12-10T09:57:05.434337Z 2021-12-10T09:57:05.356502Z 2021-12-10T09:57:05.297548Z 2021-12-10T09:57:05.235244Z 2021-12-10T09:57:05.162764Z 2021-12-10T09:57:05.105083Z 2021-12-10T09:57:05.055391Z 2021-12-10T09:57:04.998485Z 2021-12-10T09:57:04.942583Z 2021-12-10T09:57:04.902643Z 2021-12-10T09:57:04.857104Z 2021-12-10T09:57:04.806184Z 2021-12-10T09:57:04.753153Z 2021-12-10T09:57:04.703566Z 2021-12-10T09:57:04.652514Z 2021-12-10T09:57:04.604383Z 2021-12-10T09:57:04.554143Z 2021-12-10T09:57:04.505242Z 2021-12-10T09:57:04.460172Z 2021-12-10T09:57:04.413171Z 2021-12-10T09:57:04.363376Z 2021-12-10T09:57:04.319161Z 2021-12-10T09:57:04.267304Z 2021-12-10T09:57:04.210389Z 2021-12-10T09:57:04.167048Z 2021-12-10T09:57:04.126311Z 2021-12-10T09:57:04.082233Z 2021-12-10T09:57:04.024272Z 2021-12-10T09:57:03.946393Z 2021-12-10T09:57:03.869243Z 2021-12-10T09:56:43.113370Z 2021-12-10T09:57:03.801645Z 2021-12-10T09:56:43.081854Z 2021-12-10T09:56:43.046137Z 2021-12-10T09:56:43.013978Z 2021-12-10T09:56:42.980052Z 2021-12-10T09:56:42.946512Z 2021-12-10T09:56:42.903050Z 2021-12-10T09:56:42.861387Z 2021-12-10T09:56:42.814474Z 2021-12-10T09:56:42.762523Z 2021-12-10T09:56:42.720131Z 2021-12-10T09:56:42.675548Z 2021-12-10T09:56:42.632056Z 2021-12-10T09:56:42.590691Z 2021-12-10T09:56:42.557665Z 2021-12-10T09:56:42.512665Z 2021-12-10T09:56:42.460531Z 2021-12-10T09:56:42.418418Z 2021-12-10T09:56:42.384813Z 2021-12-10T09:56:42.349924Z 2021-12-10T09:56:42.305122Z 2021-12-10T09:56:42.254179Z 2021-12-10T09:56:42.209377Z 2021-12-10T09:56:42.172621Z 2021-12-10T09:56:42.139247Z 2021-12-10T09:56:42.098506Z 2021-12-10T09:56:42.063229Z 2021-12-10T09:56:41.971675Z 2021-12-10T09:56:41.904951Z 2021-12-10T09:56:41.869971Z 2021-12-10T09:56:41.831340Z 2021-12-10T09:56:41.782370Z 2021-12-10T09:56:41.730207Z 2021-12-10T09:56:41.686342Z 2021-12-10T09:56:41.646605Z 2021-12-10T09:56:41.572440Z 2021-12-10T09:56:41.495546Z 2021-12-10T09:56:41.458196Z 2021-12-10T09:56:41.421745Z 2021-12-10T09:56:41.383856Z 2021-12-10T09:56:41.340538Z 2021-12-10T09:56:41.282685Z 2021-12-10T09:56:41.245946Z 2021-12-10T09:56:41.202395Z 2021-12-10T09:56:41.165792Z 2021-12-10T09:56:41.108321Z 2021-12-10T09:56:40.964820Z 2021-12-10T09:56:40.915145Z 2021-12-10T09:56:40.873823Z 2021-12-10T09:56:34.049234Z 2021-12-10T09:56:33.943992Z 2021-12-10T09:56:33.670935Z 2021-12-10T09:56:33.455377Z 2021-12-10T09:56:33.269982Z 2021-12-10T09:56:33.120257Z 2021-12-10T09:56:32.951675Z 2021-12-10T09:56:31.697173Z 2021-12-10T09:56:31.656444Z 2021-12-10T09:56:31.597570Z", + "modified_diff": 37.5096236, + "started": "2021-12-10T09:39:14.608803Z 2021-12-10T09:39:13.357340Z 2021-12-10T09:39:14.485307Z 2021-12-10T09:39:12.916581Z 2021-12-10T09:39:13.963405Z 2021-12-10T09:39:12.551049Z 2021-12-10T09:39:12.694520Z 2021-12-10T09:39:12.268543Z 2021-12-10T09:39:12.242334Z 2021-12-10T09:39:12.009757Z 2021-12-10T09:39:07.740528Z 2021-12-10T09:39:07.479093Z 2021-12-10T09:39:07.487026Z 2021-12-10T09:39:06.887001Z 2021-12-10T09:39:07.467296Z 2021-12-10T09:39:06.386829Z 2021-12-10T09:39:06.092245Z 2021-12-10T09:39:05.764054Z 2021-12-10T09:39:05.983492Z 2021-12-10T09:39:05.421304Z 2021-12-10T09:39:05.827929Z 2021-12-10T09:39:04.911136Z 2021-12-10T09:39:05.181495Z 2021-12-10T09:39:04.682031Z 2021-12-10T09:39:02.493290Z 2021-12-10T09:39:04.698680Z 2021-12-10T09:39:02.178403Z 2021-12-10T09:39:01.872581Z 2021-12-10T09:39:01.754384Z 2021-12-10T09:39:01.591002Z 2021-12-10T09:39:01.507098Z 2021-12-10T09:39:00.251880Z 2021-12-10T09:39:00.117503Z 2021-12-10T09:38:59.618670Z 2021-12-10T09:38:58.601402Z 2021-12-10T09:39:53.558197Z 2021-12-10T09:39:53.203994Z 2021-12-10T09:39:53.311518Z 2021-12-10T09:39:52.655689Z 2021-12-10T09:39:52.635902Z 2021-12-10T09:39:52.105496Z 2021-12-10T09:39:51.754525Z 2021-12-10T09:39:45.771170Z 2021-12-10T09:39:51.613527Z 2021-12-10T09:39:44.897643Z 2021-12-10T09:39:44.954652Z 2021-12-10T09:39:44.273576Z 2021-12-10T09:39:43.740122Z 2021-12-10T09:39:43.000077Z 2021-12-10T09:39:43.096491Z 2021-12-10T09:39:42.515094Z 2021-12-10T09:39:42.130401Z 2021-12-10T09:39:41.258619Z 2021-12-10T09:39:41.261699Z 2021-12-10T09:39:40.516611Z 2021-12-10T09:39:39.972011Z 2021-12-10T09:39:39.962186Z 2021-12-10T09:39:39.313207Z 2021-12-10T09:39:39.116157Z 2021-12-10T09:39:38.633160Z 2021-12-10T09:39:38.579775Z 2021-12-10T09:39:38.222493Z 2021-12-10T09:39:37.777815Z 2021-12-10T09:39:37.632418Z 2021-12-10T09:39:37.248440Z 2021-12-10T09:39:36.872950Z 2021-12-10T09:39:36.621274Z 2021-12-10T09:39:36.405135Z 2021-12-10T09:39:36.304824Z 2021-12-10T09:39:35.883137Z 2021-12-10T09:39:35.693343Z 2021-12-10T09:39:35.567177Z 2021-12-10T09:39:35.148175Z 2021-12-10T09:39:34.627549Z 2021-12-10T09:39:34.441912Z 2021-12-10T09:39:34.021523Z 2021-12-10T09:39:33.921694Z 2021-12-10T09:39:18.631348Z 2021-12-10T09:39:33.647076Z 2021-12-10T09:39:33.373694Z 2021-12-10T09:39:20.082789Z 2021-12-10T09:39:17.917723Z 2021-12-10T09:39:19.059432Z 2021-12-10T09:39:17.692770Z 2021-12-10T09:39:18.279281Z 2021-12-10T09:39:16.753713Z 2021-12-10T09:39:17.171411Z 2021-12-10T09:39:16.278174Z 2021-12-10T09:39:16.914324Z 2021-12-10T09:39:15.908950Z 2021-12-10T09:39:16.374588Z 2021-12-10T09:39:15.556119Z 2021-12-10T09:39:16.479596Z 2021-12-10T09:39:15.296626Z 2021-12-10T09:39:15.706497Z 2021-12-10T09:39:14.732116Z 2021-12-10T09:39:16.254310Z 2021-12-10T09:39:14.178725Z 2021-12-10T09:39:15.300340Z 2021-12-10T09:39:13.618731Z 2021-12-10T09:44:43.938161Z 2021-12-10T09:44:43.706760Z 2021-12-10T09:44:43.191100Z 2021-12-10T09:44:42.730578Z 2021-12-10T09:44:42.293116Z 2021-12-10T09:44:42.247639Z 2021-12-10T09:44:41.962344Z 2021-12-10T09:44:41.566376Z 2021-12-10T09:44:41.395216Z 2021-12-10T09:44:41.437280Z 2021-12-10T09:44:40.754038Z 2021-12-10T09:44:40.339254Z 2021-12-10T09:44:40.306416Z 2021-12-10T09:44:39.967418Z 2021-12-10T09:44:39.814880Z 2021-12-10T09:44:39.304419Z 2021-12-10T09:44:39.144962Z 2021-12-10T09:44:39.135023Z 2021-12-10T09:44:38.667666Z 2021-12-10T09:44:38.597295Z 2021-12-10T09:44:38.462545Z 2021-12-10T09:44:38.210445Z 2021-12-10T09:44:37.952941Z 2021-12-10T09:44:37.796184Z 2021-12-10T09:44:37.726287Z 2021-12-10T09:44:33.465926Z 2021-12-10T09:44:32.698270Z 2021-12-10T09:44:32.343079Z 2021-12-10T09:44:31.635864Z 2021-12-10T09:44:31.128580Z 2021-12-10T09:44:30.849422Z 2021-12-10T09:44:30.865810Z 2021-12-10T09:44:30.350928Z 2021-12-10T09:44:30.098332Z 2021-12-10T09:44:29.541972Z 2021-12-10T09:44:29.409229Z 2021-12-10T09:44:37.508586Z 2021-12-10T09:44:29.157434Z 2021-12-10T09:44:28.973381Z 2021-12-10T09:44:28.590617Z 2021-12-10T09:44:28.614385Z 2021-12-10T09:44:27.810185Z 2021-12-10T09:44:28.122195Z 2021-12-10T09:44:27.615928Z 2021-12-10T09:44:27.450887Z 2021-12-10T09:44:26.824115Z 2021-12-10T09:44:26.692194Z 2021-12-10T09:44:26.414423Z 2021-12-10T09:44:26.041117Z 2021-12-10T09:44:25.712326Z 2021-12-10T09:44:25.149981Z 2021-12-10T09:44:24.831549Z 2021-12-10T09:44:24.730128Z 2021-12-10T09:44:24.350257Z 2021-12-10T09:44:24.245551Z 2021-12-10T09:44:24.059313Z 2021-12-10T09:44:23.561113Z 2021-12-10T09:44:22.654428Z 2021-12-10T09:44:22.182684Z 2021-12-10T09:44:21.733299Z 2021-12-10T09:44:21.561932Z 2021-12-10T09:44:20.930591Z 2021-12-10T09:44:20.477805Z 2021-12-10T09:44:20.174013Z 2021-12-10T09:44:19.883738Z 2021-12-10T09:44:19.565843Z 2021-12-10T09:44:19.463193Z 2021-12-10T09:44:18.958275Z 2021-12-10T09:44:18.364336Z 2021-12-10T09:44:18.318959Z 2021-12-10T09:44:17.848644Z 2021-12-10T09:44:17.725369Z 2021-12-10T09:44:17.421713Z 2021-12-10T09:44:17.305366Z 2021-12-10T09:44:16.965210Z 2021-12-10T09:44:16.779182Z 2021-12-10T09:44:16.606766Z 2021-12-10T09:44:16.399221Z 2021-12-10T09:44:05.376780Z 2021-12-10T09:44:04.715393Z 2021-12-10T09:44:03.037490Z 2021-12-10T09:44:03.377498Z 2021-12-10T09:44:01.906187Z 2021-12-10T09:44:01.676188Z 2021-12-10T09:44:00.985746Z 2021-12-10T09:44:00.986560Z 2021-12-10T09:44:00.224052Z 2021-12-10T09:44:00.168155Z 2021-12-10T09:43:59.734084Z 2021-12-10T09:43:59.487492Z 2021-12-10T09:43:58.643150Z 2021-12-10T09:43:58.485000Z 2021-12-10T09:43:58.089119Z 2021-12-10T09:43:57.914409Z 2021-12-10T09:43:57.523339Z 2021-12-10T09:43:55.024101Z 2021-12-10T09:43:54.783254Z 2021-12-10T09:43:54.512247Z 2021-12-10T09:43:54.085056Z 2021-12-10T09:43:52.644337Z 2021-12-10T09:48:55.954960Z 2021-12-10T09:48:55.713371Z 2021-12-10T09:48:55.904709Z 2021-12-10T09:48:54.854588Z 2021-12-10T09:48:54.615208Z 2021-12-10T09:48:54.647536Z 2021-12-10T09:48:54.281209Z 2021-12-10T09:48:53.984983Z 2021-12-10T09:48:53.596113Z 2021-12-10T09:48:53.222692Z 2021-12-10T09:48:52.978328Z 2021-12-10T09:48:52.496470Z 2021-12-10T09:48:52.150522Z 2021-12-10T09:48:51.835341Z 2021-12-10T09:48:51.615018Z 2021-12-10T09:48:51.058310Z 2021-12-10T09:48:50.588313Z 2021-12-10T09:48:50.187293Z 2021-12-10T09:48:49.539747Z 2021-12-10T09:48:48.831054Z 2021-12-10T09:48:48.590875Z 2021-12-10T09:48:48.257333Z 2021-12-10T09:48:47.894170Z 2021-12-10T09:48:47.647019Z 2021-12-10T09:48:47.003069Z 2021-12-10T09:48:46.586287Z 2021-12-10T09:48:46.258533Z 2021-12-10T09:48:45.997101Z 2021-12-10T09:48:45.351075Z 2021-12-10T09:48:45.137267Z 2021-12-10T09:48:44.773988Z 2021-12-10T09:48:44.128457Z 2021-12-10T09:48:44.024517Z 2021-12-10T09:48:43.721403Z 2021-12-10T09:48:43.161321Z 2021-12-10T09:48:42.765166Z 2021-12-10T09:48:42.260186Z 2021-12-10T09:48:41.956817Z 2021-12-10T09:48:41.495297Z 2021-12-10T09:48:41.029686Z 2021-12-10T09:48:40.339660Z 2021-12-10T09:48:40.005416Z 2021-12-10T09:48:39.059475Z 2021-12-10T09:48:38.469146Z 2021-12-10T09:48:38.134416Z 2021-12-10T09:48:38.042272Z 2021-12-10T09:48:37.499583Z 2021-12-10T09:48:37.255203Z 2021-12-10T09:48:36.855821Z 2021-12-10T09:48:36.575714Z 2021-12-10T09:48:36.315371Z 2021-12-10T09:48:36.358350Z 2021-12-10T09:48:35.784217Z 2021-12-10T09:48:35.749269Z 2021-12-10T09:48:35.133648Z 2021-12-10T09:48:34.919414Z 2021-12-10T09:48:34.588228Z 2021-12-10T09:48:34.391465Z 2021-12-10T09:48:34.147280Z 2021-12-10T09:48:27.536173Z 2021-12-10T09:48:33.924990Z 2021-12-10T09:48:27.437167Z 2021-12-10T09:48:33.831086Z 2021-12-10T09:48:27.187372Z 2021-12-10T09:48:26.950142Z 2021-12-10T09:48:26.634744Z 2021-12-10T09:48:26.167033Z 2021-12-10T09:48:25.800714Z 2021-12-10T09:48:25.548120Z 2021-12-10T09:48:25.594656Z 2021-12-10T09:48:24.591264Z 2021-12-10T09:48:24.428653Z 2021-12-10T09:48:24.098852Z 2021-12-10T09:48:23.849444Z 2021-12-10T09:48:23.358275Z 2021-12-10T09:48:23.370766Z 2021-12-10T09:48:22.962913Z 2021-12-10T09:48:23.024295Z 2021-12-10T09:48:22.797654Z 2021-12-10T09:48:22.608902Z 2021-12-10T09:48:22.213440Z 2021-12-10T09:48:22.056372Z 2021-12-10T09:48:21.747526Z 2021-12-10T09:48:21.619120Z 2021-12-10T09:48:21.217034Z 2021-12-10T09:48:21.172850Z 2021-12-10T09:48:20.830433Z 2021-12-10T09:48:20.462972Z 2021-12-10T09:48:20.278278Z 2021-12-10T09:48:20.035810Z 2021-12-10T09:48:19.877104Z 2021-12-10T09:48:19.683825Z 2021-12-10T09:48:19.524232Z 2021-12-10T09:48:19.341081Z 2021-12-10T09:48:11.207366Z 2021-12-10T09:48:11.444911Z 2021-12-10T09:48:10.352142Z 2021-12-10T09:48:09.950401Z 2021-12-10T09:48:09.246863Z 2021-12-10T09:48:07.035580Z 2021-12-10T09:53:04.449275Z 2021-12-10T09:53:03.897217Z 2021-12-10T09:53:03.588645Z 2021-12-10T09:53:03.400290Z 2021-12-10T09:53:03.182850Z 2021-12-10T09:53:02.813163Z 2021-12-10T09:53:01.767417Z 2021-12-10T09:53:01.342556Z 2021-12-10T09:53:00.889051Z 2021-12-10T09:53:00.672021Z 2021-12-10T09:53:00.035181Z 2021-12-10T09:52:59.820541Z 2021-12-10T09:52:59.182700Z 2021-12-10T09:52:59.014355Z 2021-12-10T09:52:58.167144Z 2021-12-10T09:52:57.700414Z 2021-12-10T09:52:56.975656Z 2021-12-10T09:52:56.552443Z 2021-12-10T09:52:56.051876Z 2021-12-10T09:52:55.457192Z 2021-12-10T09:52:54.567108Z 2021-12-10T09:52:54.801756Z 2021-12-10T09:52:53.956967Z 2021-12-10T09:52:53.871480Z 2021-12-10T09:52:53.409146Z 2021-12-10T09:52:53.533850Z 2021-12-10T09:52:52.743251Z 2021-12-10T09:52:52.513689Z 2021-12-10T09:52:52.054255Z 2021-12-10T09:52:51.706298Z 2021-12-10T09:52:51.377118Z 2021-12-10T09:52:50.648234Z 2021-12-10T09:52:50.715845Z 2021-12-10T09:52:50.575480Z 2021-12-10T09:52:50.089216Z 2021-12-10T09:52:49.779165Z 2021-12-10T09:52:49.429059Z 2021-12-10T09:52:49.164351Z 2021-12-10T09:52:48.821399Z 2021-12-10T09:52:48.886172Z 2021-12-10T09:52:48.061620Z 2021-12-10T09:52:47.962642Z 2021-12-10T09:52:47.611114Z 2021-12-10T09:52:47.485380Z 2021-12-10T09:52:47.543131Z 2021-12-10T09:52:47.097874Z 2021-12-10T09:52:47.080882Z 2021-12-10T09:52:46.829266Z 2021-12-10T09:52:46.613184Z 2021-12-10T09:52:46.353111Z 2021-12-10T09:52:46.346029Z 2021-12-10T09:52:45.991783Z 2021-12-10T09:52:37.473352Z 2021-12-10T09:52:45.863133Z 2021-12-10T09:52:36.647664Z 2021-12-10T09:52:36.438882Z 2021-12-10T09:52:36.071622Z 2021-12-10T09:52:35.803416Z 2021-12-10T09:52:35.419514Z 2021-12-10T09:52:35.258290Z 2021-12-10T09:52:35.001843Z 2021-12-10T09:52:34.619028Z 2021-12-10T09:52:34.354198Z 2021-12-10T09:52:34.500330Z 2021-12-10T09:52:33.909427Z 2021-12-10T09:52:33.727322Z 2021-12-10T09:52:33.313129Z 2021-12-10T09:52:32.964470Z 2021-12-10T09:52:32.499122Z 2021-12-10T09:52:32.531576Z 2021-12-10T09:52:31.837136Z 2021-12-10T09:52:31.667292Z 2021-12-10T09:52:31.176224Z 2021-12-10T09:52:31.262127Z 2021-12-10T09:52:30.488700Z 2021-12-10T09:52:30.237741Z 2021-12-10T09:52:29.877188Z 2021-12-10T09:52:29.959991Z 2021-12-10T09:52:29.738023Z 2021-12-10T09:52:29.700463Z 2021-12-10T09:52:29.279014Z 2021-12-10T09:52:29.112179Z 2021-12-10T09:52:28.785108Z 2021-12-10T09:52:28.542286Z 2021-12-10T09:52:28.348931Z 2021-12-10T09:52:28.111739Z 2021-12-10T09:52:28.094518Z 2021-12-10T09:52:27.883311Z 2021-12-10T09:52:27.500681Z 2021-12-10T09:52:27.475235Z 2021-12-10T09:52:27.136493Z 2021-12-10T09:52:27.212866Z 2021-12-10T09:52:26.940388Z 2021-12-10T09:52:26.863502Z 2021-12-10T09:52:18.640117Z 2021-12-10T09:52:18.240011Z 2021-12-10T09:52:17.722725Z 2021-12-10T09:52:17.431384Z 2021-12-10T09:52:17.016291Z 2021-12-10T09:52:16.774796Z 2021-12-10T09:57:18.389274Z 2021-12-10T09:57:18.268340Z 2021-12-10T09:57:18.333635Z 2021-12-10T09:57:17.822869Z 2021-12-10T09:57:17.909292Z 2021-12-10T09:57:17.346147Z 2021-12-10T09:57:16.850871Z 2021-12-10T09:57:16.614057Z 2021-12-10T09:57:16.345140Z 2021-12-10T09:57:16.131030Z 2021-12-10T09:57:15.572013Z 2021-12-10T09:57:15.137279Z 2021-12-10T09:57:15.006104Z 2021-12-10T09:57:14.709902Z 2021-12-10T09:57:14.429266Z 2021-12-10T09:57:14.274781Z 2021-12-10T09:57:13.852798Z 2021-12-10T09:57:13.490923Z 2021-12-10T09:57:13.399799Z 2021-12-10T09:57:12.900341Z 2021-12-10T09:57:12.583730Z 2021-12-10T09:57:11.868822Z 2021-12-10T09:57:11.631985Z 2021-12-10T09:57:11.069773Z 2021-12-10T09:57:10.692175Z 2021-12-10T09:57:10.521966Z 2021-12-10T09:57:10.235156Z 2021-12-10T09:57:09.999453Z 2021-12-10T09:57:09.752126Z 2021-12-10T09:57:09.753839Z 2021-12-10T09:57:09.599141Z 2021-12-10T09:57:09.437464Z 2021-12-10T09:57:09.470250Z 2021-12-10T09:57:08.751116Z 2021-12-10T09:57:08.545512Z 2021-12-10T09:57:08.333111Z 2021-12-10T09:57:07.925133Z 2021-12-10T09:57:07.729614Z 2021-12-10T09:57:07.338227Z 2021-12-10T09:57:07.259669Z 2021-12-10T09:56:58.397670Z 2021-12-10T09:57:06.668195Z 2021-12-10T09:56:58.225601Z 2021-12-10T09:56:58.083272Z 2021-12-10T09:56:57.896120Z 2021-12-10T09:56:57.526658Z 2021-12-10T09:56:57.004780Z 2021-12-10T09:56:56.255281Z 2021-12-10T09:56:55.834136Z 2021-12-10T09:56:55.419860Z 2021-12-10T09:56:55.108098Z 2021-12-10T09:56:54.739430Z 2021-12-10T09:56:54.865550Z 2021-12-10T09:56:54.149155Z 2021-12-10T09:56:53.638539Z 2021-12-10T09:56:53.149855Z 2021-12-10T09:56:52.790970Z 2021-12-10T09:56:52.451179Z 2021-12-10T09:56:52.370014Z 2021-12-10T09:56:52.012235Z 2021-12-10T09:56:52.020016Z 2021-12-10T09:56:51.401943Z 2021-12-10T09:56:50.917384Z 2021-12-10T09:56:50.378900Z 2021-12-10T09:56:49.939562Z 2021-12-10T09:56:49.829988Z 2021-12-10T09:56:49.476982Z 2021-12-10T09:56:49.258942Z 2021-12-10T09:56:48.985419Z 2021-12-10T09:56:48.591242Z 2021-12-10T09:56:48.190262Z 2021-12-10T09:56:48.093418Z 2021-12-10T09:56:48.054000Z 2021-12-10T09:56:47.776490Z 2021-12-10T09:56:47.417908Z 2021-12-10T09:56:47.276535Z 2021-12-10T09:56:46.853419Z 2021-12-10T09:56:46.350312Z 2021-12-10T09:56:46.165368Z 2021-12-10T09:56:45.985716Z 2021-12-10T09:56:45.440741Z 2021-12-10T09:56:45.386679Z 2021-12-10T09:56:45.005445Z 2021-12-10T09:56:44.615771Z 2021-12-10T09:56:44.328261Z 2021-12-10T09:56:44.307557Z 2021-12-10T09:56:44.161137Z 2021-12-10T09:56:43.714048Z 2021-12-10T09:56:43.536039Z 2021-12-10T09:56:43.533353Z 2021-12-10T09:56:37.527327Z 2021-12-10T09:56:37.536294Z 2021-12-10T09:56:36.897702Z 2021-12-10T09:56:36.099252Z 2021-12-10T09:56:35.587317Z 2021-12-10T09:56:34.883750Z 2021-12-10T09:56:34.439243Z 2021-12-10T09:56:32.217720Z 2021-12-10T09:56:32.218974Z 2021-12-10T09:56:31.915116Z", + "started_diff": 49.86372719999999, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 92, 91, 90, 88, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 73, 72, 71, 70, 69, 67, 66, 65, 64, 63, 62, 61, 60, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 150, 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 131, 130, 129, 127, 126, 125, 124, 123, 121, 120, 118, 117, 116, 114, 112, 111, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472]" + }, + "started": "2021-12-10T10:22:46.367199+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-api.json new file mode 100644 index 00000000..5b995373 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-10T14:41:51.071212+00:00", + "golden": "true", + "id": "run-2021-12-10T10_54_54_920_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2361.7333333333336, + "max": 24559.6, + "mean": 5325.811472464644, + "median": 3377.0, + "min": 1320.1333333333332, + "non_zero_mean": 5325.811472464644, + "non_zero_median": 3377.0, + "percentile25": 2762.0, + "percentile75": 5123.733333333334, + "percentile90": 13111.733333333334, + "percentile99": 23375.219872452944, + "percentile999": 24458.460000000003, + "range": 23239.466666666667, + "samples": 261, + "stdev": 4786.474170593931, + "sum": 1390036.794313272 + } + }, + "cpu": { + "system": { + "iqr": 0.18466666666666925, + "max": 1.69466666666666, + "mean": 0.27017110606510664, + "median": 0.11266666666667031, + "min": 0.032000000000001215, + "non_zero_mean": 0.27017110606510664, + "non_zero_median": 0.11266666666667031, + "percentile25": 0.06066666666666075, + "percentile75": 0.24533333333333002, + "percentile90": 0.9126666666666627, + "percentile99": 1.6423999999999905, + "percentile999": 1.683906527120411, + "range": 1.662666666666659, + "samples": 261, + "stdev": 0.38563612876669656, + "sum": 70.51465868299283 + }, + "user": { + "iqr": 0.5553333333333437, + "max": 6.2873333333333, + "mean": 1.369650022550012, + "median": 0.7779999999999746, + "min": 0.38200000000004664, + "non_zero_mean": 1.369650022550012, + "non_zero_median": 0.7779999999999746, + "percentile25": 0.59066666666666, + "percentile75": 1.1460000000000037, + "percentile90": 3.91266666666667, + "percentile99": 6.217199999999999, + "percentile999": 6.276759999999989, + "range": 5.905333333333253, + "samples": 261, + "stdev": 1.4729740627728478, + "sum": 357.47865588555294 + } + }, + "entropy_available_bits": { + "iqr": 1.866666666666667, + "max": 213.66666666666666, + "mean": 20.15453370851256, + "median": 1.7999999999999998, + "min": 0.2, + "non_zero_mean": 20.15453370851256, + "non_zero_median": 1.7999999999999998, + "percentile25": 0.8, + "percentile75": 2.666666666666667, + "percentile90": 7.2, + "percentile99": 212.42666666666668, + "percentile999": 213.56266666666667, + "range": 213.46666666666667, + "samples": 261, + "stdev": 56.061293433624066, + "sum": 5260.3332979217785 + }, + "memory": { + "used": { + "iqr": 293036032.0, + "max": 16026333184.0, + "mean": 6370321152.980843, + "median": 5843062784.0, + "min": 5337042944.0, + "non_zero_mean": 6370321152.980843, + "non_zero_median": 5843062784.0, + "percentile25": 5727338496.0, + "percentile75": 6020374528.0, + "percentile90": 6810714112.0, + "percentile99": 15445151743.999994, + "percentile999": 15940484628.480003, + "range": 10689290240.0, + "samples": 261, + "stdev": 1843147738.1450174, + "sum": 1662653820928.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 16.4, + "mean": 0.17496807151979565, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6.523809523809524, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 6.106666666666456, + "percentile999": 15.186666666666708, + "range": 16.4, + "samples": 261, + "stdev": 1.4506774312921573, + "sum": 45.66666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 83.93333333333334, + "max": 1685.6000000000001, + "mean": 119.37727987939039, + "median": 29.200000000000003, + "min": 0.0, + "non_zero_mean": 120.29911215645132, + "non_zero_median": 29.266666666666666, + "percentile25": 1.3333333333333335, + "percentile75": 85.26666666666667, + "percentile90": 343.73333333333335, + "percentile99": 1179.507078249661, + "percentile999": 1585.6386666666701, + "range": 1685.6000000000001, + "samples": 261, + "stdev": 258.5609636504811, + "sum": 31157.470048520878 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 941.9333333333334, + "max": 2948.733333333333, + "mean": 1620.5420178799488, + "median": 1567.2666666666667, + "min": 360.26666666666665, + "non_zero_mean": 1620.5420178799488, + "non_zero_median": 1567.2666666666667, + "percentile25": 1152.9333333333334, + "percentile75": 2094.866666666667, + "percentile90": 2442.733333333333, + "percentile99": 2891.2933333333326, + "percentile999": 2938.576, + "range": 2588.4666666666662, + "samples": 261, + "stdev": 593.7772244421042, + "sum": 422961.46666666673 + } + }, + "cpu": { + "system": { + "iqr": 0.047333333333331964, + "max": 0.2553333333333323, + "mean": 0.08582886334610473, + "median": 0.0726666666666669, + "min": 0.014666666666666592, + "non_zero_mean": 0.08582886334610473, + "non_zero_median": 0.0726666666666669, + "percentile25": 0.05466666666666621, + "percentile75": 0.10199999999999818, + "percentile90": 0.15733333333333233, + "percentile99": 0.2181333333333327, + "percentile999": 0.24822666666666596, + "range": 0.2406666666666657, + "samples": 261, + "stdev": 0.04529530457668326, + "sum": 22.401333333333323 + }, + "user": { + "iqr": 0.12133333333333857, + "max": 1.027333333333339, + "mean": 0.20349169859514685, + "median": 0.19599999999999984, + "min": 0.016666666666666666, + "non_zero_mean": 0.20349169859514685, + "non_zero_median": 0.19599999999999984, + "percentile25": 0.13466666666666166, + "percentile75": 0.2560000000000002, + "percentile90": 0.3333333333333333, + "percentile99": 0.4741333333333312, + "percentile999": 0.9080800000000069, + "range": 1.0106666666666724, + "samples": 261, + "stdev": 0.10778369659338316, + "sum": 53.11133333333331 + } + }, + "entropy_available_bits": { + "iqr": 1.4666666666666666, + "max": 211.66666666666666, + "mean": 11.333077905491699, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 20.978250591016547, + "non_zero_median": 1.4666666666666666, + "percentile25": 0.0, + "percentile75": 1.4666666666666666, + "percentile90": 3.2666666666666666, + "percentile99": 211.34666666666666, + "percentile999": 211.632, + "range": 211.66666666666666, + "samples": 261, + "stdev": 45.83794214518629, + "sum": 2957.933333333335 + }, + "memory": { + "used": { + "iqr": 18612224.0, + "max": 1045762048.0, + "mean": 428838331.34099615, + "median": 388382720.0, + "min": 368041984.0, + "non_zero_mean": 428838331.34099615, + "non_zero_median": 388382720.0, + "percentile25": 382144512.0, + "percentile75": 400756736.0, + "percentile90": 505704448.0, + "percentile99": 1006919679.9999999, + "percentile999": 1044924989.44, + "range": 677720064.0, + "samples": 261, + "stdev": 125386021.11136097, + "sum": 111926804480.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6553.6, + "max": 59393697.0, + "mean": 1126345.6722860793, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3062252.296527778, + "non_zero_median": 10649.599999999999, + "percentile25": 0.0, + "percentile75": 6553.6, + "percentile90": 38229.333333333336, + "percentile99": 57539792.78666666, + "percentile999": 59301684.456, + "range": 59393697.0, + "samples": 261, + "stdev": 7992597.219794124, + "sum": 293976220.46666676 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 313.870584, + "mean": 4.073152551724138, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 21.69577175510204, + "non_zero_median": 0.018555, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.014159, + "percentile99": 155.62741419999963, + "percentile999": 290.4700754400008, + "range": 313.870584, + "samples": 261, + "stdev": 28.587554594872824, + "sum": 1063.0928160000003 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.226314, + "max": 433.869781, + "mean": 25.054364915708813, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 55.89050635042735, + "non_zero_median": 0.766382, + "percentile25": 0.0, + "percentile75": 0.226314, + "percentile90": 73.88802, + "percentile99": 394.88741499999963, + "percentile999": 429.9736724200001, + "range": 433.869781, + "samples": 261, + "stdev": 79.56623052866551, + "sum": 6539.1892430000025 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.733333333333333, + "mean": 0.16398467432950192, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.292307692307692, + "non_zero_median": 3.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.6933333333333316, + "percentile999": 4.664000000000002, + "range": 4.733333333333333, + "samples": 261, + "stdev": 0.7348565484315749, + "sum": 42.800000000000004 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9994.866666666667, + "mean": 401.28863346104725, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8056.641025641025, + "non_zero_median": 7114.8, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9974.706666666667, + "percentile999": 9994.728000000001, + "range": 9994.866666666667, + "samples": 261, + "stdev": 1796.9318054668936, + "sum": 104736.33333333333 + }, + "pg_stat_database_blks_hit": { + "iqr": 19506.533333333333, + "max": 218748.46666666667, + "mean": 26128.20178799489, + "median": 25858.4, + "min": 2482.266666666667, + "non_zero_mean": 26128.20178799489, + "non_zero_median": 25858.4, + "percentile25": 14223.666666666666, + "percentile75": 33730.2, + "percentile90": 46029.4, + "percentile99": 70299.50666666614, + "percentile999": 208661.24666666702, + "range": 216266.2, + "samples": 261, + "stdev": 21403.314500815857, + "sum": 6819460.666666666 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5333333333333333, + "max": 215.46666666666667, + "mean": 1.4209450830140484, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7830128205128206, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 2.066666666666667, + "percentile99": 4.319999999999997, + "percentile999": 161.23066666666858, + "range": 215.46666666666667, + "samples": 261, + "stdev": 13.335723015050664, + "sum": 370.86666666666616 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 261, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5333333333333333, + "max": 215.46666666666667, + "mean": 1.4209450830140484, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7830128205128206, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 2.066666666666667, + "percentile99": 4.319999999999997, + "percentile999": 161.23066666666858, + "range": 215.46666666666667, + "samples": 261, + "stdev": 13.335723015050664, + "sum": 370.86666666666616 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.4, + "max": 4007.0666666666666, + "mean": 41.58033205619412, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 102.38176100628931, + "non_zero_median": 16.03333333333333, + "percentile25": 0.0, + "percentile75": 2.4, + "percentile90": 59.53333333333333, + "percentile99": 81.78666666666655, + "percentile999": 4006.061333333333, + "range": 4007.0666666666666, + "samples": 261, + "stdev": 349.6812915785748, + "sum": 10852.466666666669 + }, + "pg_stat_database_tup_fetched": { + "iqr": 10888.2, + "max": 355847.5333333333, + "mean": 14648.788505747127, + "median": 13854.733333333334, + "min": 1246.8666666666666, + "non_zero_mean": 14648.788505747127, + "non_zero_median": 13854.733333333334, + "percentile25": 6775.533333333334, + "percentile75": 17663.733333333334, + "percentile90": 25178.466666666667, + "percentile99": 36088.079999999885, + "percentile999": 273784.4613333362, + "range": 354600.6666666667, + "samples": 261, + "stdev": 22836.958103666886, + "sum": 3823333.800000001 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.8, + "max": 8005.333333333333, + "mean": 46.67407407407407, + "median": 2.533333333333333, + "min": 0.0, + "non_zero_mean": 56.66015503875969, + "non_zero_median": 3.6666666666666665, + "percentile25": 1.0666666666666667, + "percentile75": 9.866666666666667, + "percentile90": 67.66666666666667, + "percentile99": 146.79999999999995, + "percentile999": 5965.841333333405, + "range": 8005.333333333333, + "samples": 261, + "stdev": 495.67228468834844, + "sum": 12181.933333333322 + }, + "pg_stat_database_tup_returned": { + "iqr": 39568.933333333334, + "max": 1190695.0, + "mean": 51025.41992337165, + "median": 42091.73333333333, + "min": 3486.6, + "non_zero_mean": 51025.41992337165, + "non_zero_median": 42091.73333333333, + "percentile25": 21188.466666666667, + "percentile75": 60757.4, + "percentile90": 89673.0, + "percentile99": 122839.62666666662, + "percentile999": 1097406.2720000031, + "range": 1187208.4, + "samples": 261, + "stdev": 91086.98809595226, + "sum": 13317634.60000001 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.266666666666667, + "max": 61.46666666666667, + "mean": 10.471775223499362, + "median": 7.4, + "min": 0.13333333333333333, + "non_zero_mean": 10.471775223499362, + "non_zero_median": 7.4, + "percentile25": 3.533333333333333, + "percentile75": 15.8, + "percentile90": 23.133333333333333, + "percentile99": 39.89333333333305, + "percentile999": 58.24266666666678, + "range": 61.333333333333336, + "samples": 261, + "stdev": 9.64165860419666, + "sum": 2733.133333333333 + }, + "pg_stat_database_xact_commit": { + "iqr": 133.8, + "max": 727.0666666666667, + "mean": 124.1765006385696, + "median": 90.13333333333334, + "min": 5.2, + "non_zero_mean": 124.1765006385696, + "non_zero_median": 90.13333333333334, + "percentile25": 40.6, + "percentile75": 174.4, + "percentile90": 266.53333333333336, + "percentile99": 567.5066666666667, + "percentile999": 716.9440000000004, + "range": 721.8666666666667, + "samples": 261, + "stdev": 125.02381923969537, + "sum": 32410.066666666666 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.1269476372924649, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.1269476372924649, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 261, + "stdev": 0.1452734153116778, + "sum": 33.13333333333333 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 261, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 29.866666666666664, + "max": 136.06666666666666, + "mean": 39.45057471264368, + "median": 32.4, + "min": 1.7333333333333334, + "non_zero_mean": 39.45057471264368, + "non_zero_median": 32.4, + "percentile25": 22.933333333333334, + "percentile75": 52.8, + "percentile90": 75.06666666666666, + "percentile99": 107.23999999999987, + "percentile999": 130.72800000000018, + "range": 134.33333333333334, + "samples": 261, + "stdev": 24.226118487618425, + "sum": 10296.599999999997 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-10T13:36:40.852160+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-10T10_54_54_920_0000", + "ended": "2021-12-10T14:41:45.847665+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.023098767000647058, + "iterations": 1, + "max": 0.5592759930004831, + "mean": 0.5416496641333045, + "median": 0.5407742500001405, + "min": 0.5251307460002863, + "rounds": 15, + "stddev": 0.011925196271775674 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.021529032000444204, + "iterations": 1, + "max": 0.5589888109998356, + "mean": 0.5330798591333709, + "median": 0.5376980090004508, + "min": 0.5126987920002648, + "rounds": 15, + "stddev": 0.014184021040250065 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.03974457699951017, + "iterations": 1, + "max": 0.7570769350004412, + "mean": 0.43233382610014814, + "median": 0.39940954199983025, + "min": 0.36940886699994735, + "rounds": 10, + "stddev": 0.11555564671490584 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.007236562999878515, + "iterations": 1, + "max": 0.27764500099965517, + "mean": 0.2587837540399869, + "median": 0.2588634170001569, + "min": 0.24286019800001668, + "rounds": 100, + "stddev": 0.006385640620432386 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.014397288500276773, + "iterations": 1, + "max": 0.5387868660000095, + "mean": 0.4697637210499852, + "median": 0.4690110975002426, + "min": 0.43951696699969034, + "rounds": 100, + "stddev": 0.013436429180190512 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.04830066900012753, + "iterations": 1, + "max": 1.4018216019994725, + "mean": 1.3289218244999574, + "median": 1.322299595000004, + "min": 1.273223262999636, + "rounds": 10, + "stddev": 0.03864745470096997 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.002352055999381264, + "iterations": 1, + "max": 0.07729278700026043, + "mean": 0.07276209039991954, + "median": 0.07252918199992564, + "min": 0.07008319299984578, + "rounds": 15, + "stddev": 0.0018795627455576206 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0007729560002189828, + "iterations": 1, + "max": 0.0802592559994082, + "mean": 0.07469858742862405, + "median": 0.07440930299981119, + "min": 0.07206025899995439, + "rounds": 14, + "stddev": 0.0020094468827414635 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.017011010749911293, + "iterations": 1, + "max": 0.11635118899994268, + "mean": 0.1018101184546884, + "median": 0.09643983600017236, + "min": 0.09045027900083369, + "rounds": 11, + "stddev": 0.009789700680921453 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.00415279000026203, + "iterations": 1, + "max": 0.2139945369999623, + "mean": 0.1203292419999343, + "median": 0.11042180250024103, + "min": 0.10617176199957612, + "rounds": 10, + "stddev": 0.03301354243236363 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.015446379000422894, + "iterations": 1, + "max": 0.2853413719994933, + "mean": 0.2729412409000361, + "median": 0.27249262950044795, + "min": 0.25842552999984036, + "rounds": 10, + "stddev": 0.009027773376454369 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.0034190820006188005, + "iterations": 1, + "max": 0.22790756399990642, + "mean": 0.1269002478999937, + "median": 0.11604048050003257, + "min": 0.11368812599994271, + "rounds": 10, + "stddev": 0.03553059331135979 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.002335050000510819, + "iterations": 1, + "max": 0.07612322000022687, + "mean": 0.07356911621426272, + "median": 0.07331298549979692, + "min": 0.07058969900026568, + "rounds": 14, + "stddev": 0.001650589123075725 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.012254802999450476, + "iterations": 1, + "max": 0.6513067680007225, + "mean": 0.6362623698999414, + "median": 0.636174132999713, + "min": 0.6197488340003474, + "rounds": 10, + "stddev": 0.010072236252360419 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0021846565002761054, + "iterations": 1, + "max": 0.10082297400003881, + "mean": 0.09529183209096201, + "median": 0.09479938300046342, + "min": 0.09313676900001155, + "rounds": 11, + "stddev": 0.0024384704550224866 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.0038879440016899025, + "iterations": 1, + "max": 0.11158971300028497, + "mean": 0.10760506930000702, + "median": 0.10737500300001557, + "min": 0.10354424100023607, + "rounds": 10, + "stddev": 0.002437541514345296 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.0063909509999575675, + "iterations": 1, + "max": 0.19073458500042761, + "mean": 0.17495359519998602, + "median": 0.17347097150013724, + "min": 0.1691771030000382, + "rounds": 10, + "stddev": 0.006285086979978148 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.002819476000695431, + "iterations": 1, + "max": 0.08194459599963011, + "mean": 0.07398244692857199, + "median": 0.07325953400049912, + "min": 0.07057029099996726, + "rounds": 14, + "stddev": 0.0032787994508111703 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.0008648192495002149, + "iterations": 1, + "max": 0.07889781500034587, + "mean": 0.07590471792298978, + "median": 0.07609896899975865, + "min": 0.07269168900074874, + "rounds": 13, + "stddev": 0.0015374328430431507 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.01082763199974579, + "iterations": 1, + "max": 0.37430467899957875, + "mean": 0.36509803769995414, + "median": 0.36808198399967296, + "min": 0.34738448100051755, + "rounds": 10, + "stddev": 0.008250639186897397 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.019605919999776233, + "iterations": 1, + "max": 0.4081518020002477, + "mean": 0.38175196840002174, + "median": 0.37673322349974114, + "min": 0.3623341829998026, + "rounds": 10, + "stddev": 0.01414428092269595 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.0022007312497862586, + "iterations": 1, + "max": 0.09610537000025943, + "mean": 0.09404605936362548, + "median": 0.09406438400037587, + "min": 0.09156066000014107, + "rounds": 11, + "stddev": 0.0014466697563547911 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.002652178500284208, + "iterations": 1, + "max": 0.08816555500015966, + "mean": 0.08606068041672188, + "median": 0.08675998150010855, + "min": 0.08214736999980232, + "rounds": 12, + "stddev": 0.0020685894817426237 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.0108925000004092, + "iterations": 1, + "max": 0.18419976199947996, + "mean": 0.1755729354998948, + "median": 0.175514905, + "min": 0.16818790799970884, + "rounds": 10, + "stddev": 0.005835141763290966 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.001561202999255329, + "iterations": 1, + "max": 0.07300541600034194, + "mean": 0.0711456695000509, + "median": 0.0709418035003182, + "min": 0.06889819600019109, + "rounds": 14, + "stddev": 0.0011329330110410782 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.0034377950005364255, + "iterations": 1, + "max": 0.20840388300075574, + "mean": 0.20623658469994552, + "median": 0.20696769449978092, + "min": 0.2026874940002017, + "rounds": 10, + "stddev": 0.0019380114026846117 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0025617374999455933, + "iterations": 1, + "max": 0.08213264000005438, + "mean": 0.07913829930776746, + "median": 0.07900976900054957, + "min": 0.07739755799957493, + "rounds": 13, + "stddev": 0.0015193596640426855 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.015136971000174526, + "iterations": 1, + "max": 0.36599831599960453, + "mean": 0.35244514309988517, + "median": 0.3521049195001069, + "min": 0.3416453999998339, + "rounds": 10, + "stddev": 0.00859084437775954 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.008811400000922731, + "iterations": 1, + "max": 0.24387435599965102, + "mean": 0.22901243429996612, + "median": 0.2265592599997035, + "min": 0.21937515700028598, + "rounds": 10, + "stddev": 0.007277148376660576 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.005680813000253693, + "iterations": 1, + "max": 0.28164732000004733, + "mean": 0.1772695264999129, + "median": 0.16617027749953195, + "min": 0.1619447510001919, + "rounds": 10, + "stddev": 0.03677274644939308 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.12316649700005655, + "iterations": 1, + "max": 0.3248150760000499, + "mean": 0.1677921013001651, + "median": 0.11437713300028918, + "min": 0.10358520799945836, + "rounds": 10, + "stddev": 0.07989888402894409 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.0017737784999098949, + "iterations": 1, + "max": 0.0749072429998705, + "mean": 0.07124374866671132, + "median": 0.07117767900035687, + "min": 0.06796643000052427, + "rounds": 15, + "stddev": 0.0017401662338938073 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.004587528000229213, + "iterations": 1, + "max": 0.1016124560001117, + "mean": 0.0968015042000843, + "median": 0.09701134000033562, + "min": 0.09111104000021442, + "rounds": 10, + "stddev": 0.003090764387438269 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0032425609997517313, + "iterations": 1, + "max": 0.07471213000007992, + "mean": 0.07279575121427635, + "median": 0.07286354150028274, + "min": 0.06987998800013884, + "rounds": 14, + "stddev": 0.0016506688492821065 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.0028056060000380967, + "iterations": 1, + "max": 0.12681434800015268, + "mean": 0.12145068459994945, + "median": 0.12106894499993359, + "min": 0.11933249100002286, + "rounds": 10, + "stddev": 0.0022231092980881093 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0019856850003634463, + "iterations": 1, + "max": 0.07506241200007935, + "mean": 0.07226588471413768, + "median": 0.07179994850002913, + "min": 0.06992873699982738, + "rounds": 14, + "stddev": 0.0014195604149690546 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.010269692999827384, + "iterations": 1, + "max": 0.6935410280002543, + "mean": 0.5872875236999789, + "median": 0.5786839654992946, + "min": 0.563905848000104, + "rounds": 10, + "stddev": 0.03775497305885635 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.0939452969996637, + "iterations": 1, + "max": 0.7504123700000491, + "mean": 0.6563988816999882, + "median": 0.6318717154999831, + "min": 0.6141148729993802, + "rounds": 10, + "stddev": 0.05278908773755864 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.007379498999398493, + "iterations": 1, + "max": 0.2855861569996705, + "mean": 0.27806570519996965, + "median": 0.2769919685001696, + "min": 0.2701734709999073, + "rounds": 10, + "stddev": 0.0052402503773767845 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0027671580000969698, + "iterations": 1, + "max": 0.08141144899946084, + "mean": 0.07458722935708854, + "median": 0.0742505030002576, + "min": 0.07140034599979117, + "rounds": 14, + "stddev": 0.00246426586370737 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.001836284999626514, + "iterations": 1, + "max": 0.07389131000036286, + "mean": 0.0725808639999741, + "median": 0.0729242739998881, + "min": 0.07008533000043826, + "rounds": 14, + "stddev": 0.0011940672148414296 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0016993919998640195, + "iterations": 1, + "max": 0.07911618500020268, + "mean": 0.07403270307135504, + "median": 0.07322926300003019, + "min": 0.07189179299984971, + "rounds": 14, + "stddev": 0.0022621293019262885 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0018318900001759175, + "iterations": 1, + "max": 0.0798266510000758, + "mean": 0.07362399707134111, + "median": 0.07307618549975814, + "min": 0.07104270800027734, + "rounds": 14, + "stddev": 0.002116468717075264 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.006164977000480576, + "iterations": 1, + "max": 0.32198812299975543, + "mean": 0.30241293120016055, + "median": 0.30216348800013293, + "min": 0.29163678100030666, + "rounds": 10, + "stddev": 0.00845917737284277 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.008802378999462235, + "iterations": 1, + "max": 0.23630298100033542, + "mean": 0.22731818870015558, + "median": 0.2272806885002865, + "min": 0.2187445640001897, + "rounds": 10, + "stddev": 0.005256007624114495 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.02342994999980874, + "iterations": 1, + "max": 0.9601474819992291, + "mean": 0.8623380019998876, + "median": 0.9324023135000061, + "min": 0.5595817089997581, + "rounds": 10, + "stddev": 0.15892202839314934 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.11294906900002388, + "iterations": 1, + "max": 8.810698504999891, + "mean": 8.6983118784, + "median": 8.69198133250029, + "min": 8.616311404000044, + "rounds": 10, + "stddev": 0.06324312739505326 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.027739357000427844, + "iterations": 1, + "max": 1.1443161520001013, + "mean": 1.0477748215000247, + "median": 1.0372025289998419, + "min": 1.019931830999667, + "rounds": 10, + "stddev": 0.03624596588738385 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.3024568660002842, + "iterations": 1, + "max": 9.321020946000317, + "mean": 9.0157467463001, + "median": 8.965540175000115, + "min": 8.803924202999951, + "rounds": 10, + "stddev": 0.1778776754837609 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.42933772500055056, + "iterations": 1, + "max": 1.6853027810002459, + "mean": 1.2210885227999824, + "median": 1.0848891129999174, + "min": 1.006473494000602, + "rounds": 10, + "stddev": 0.24721266234699854 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.02396777000012662, + "iterations": 1, + "max": 0.4141404279998824, + "mean": 0.3977423851999447, + "median": 0.40133926999988034, + "min": 0.3686065530000633, + "rounds": 10, + "stddev": 0.014504106458440356 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.10543388400219555, + "iterations": 1, + "max": 11.571069843999794, + "mean": 11.328826960599873, + "median": 11.362784706499951, + "min": 10.943510394999976, + "rounds": 10, + "stddev": 0.17902879567744823 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.010281204000420985, + "iterations": 1, + "max": 0.42268350399990595, + "mean": 0.40219828729987056, + "median": 0.4032256929999676, + "min": 0.3794532899992191, + "rounds": 10, + "stddev": 0.011632223236692892 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.2293771940003353, + "iterations": 1, + "max": 0.9857044190002853, + "mean": 0.5962121528000353, + "median": 0.5249225579996164, + "min": 0.4078733070000453, + "rounds": 10, + "stddev": 0.1824578875941045 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.11254347400063125, + "iterations": 1, + "max": 12.150213925999196, + "mean": 11.665801566399569, + "median": 11.622528058499483, + "min": 11.46841189999941, + "rounds": 10, + "stddev": 0.1932796093038581 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.13221333600085927, + "iterations": 1, + "max": 7.9210713140000735, + "mean": 7.71300436069987, + "median": 7.687471834499775, + "min": 7.623704827999973, + "rounds": 10, + "stddev": 0.09281543665505534 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 250.7478301759984, + "mean": 250.7478301759984, + "median": 250.7478301759984, + "min": 250.7478301759984, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 393.3395729540007, + "mean": 393.3395729540007, + "median": 393.3395729540007, + "min": 393.3395729540007, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.00786180900013278, + "iterations": 1, + "max": 0.2630178730005355, + "mean": 0.2558333268001661, + "median": 0.25636049800050387, + "min": 0.24898585099981574, + "rounds": 10, + "stddev": 0.004950289950756489 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.011096098000052734, + "iterations": 1, + "max": 0.7747057060005318, + "mean": 0.6600484623002558, + "median": 0.6482233890001226, + "min": 0.6342530160000024, + "rounds": 10, + "stddev": 0.040910484233085825 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.041772646001845715, + "iterations": 1, + "max": 9.658103790001405, + "mean": 9.587479716599592, + "median": 9.57370869099941, + "min": 9.543821174998811, + "rounds": 10, + "stddev": 0.03361484857520894 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.48357988800034946, + "iterations": 1, + "max": 3.7798363350002546, + "mean": 3.0747490525000103, + "median": 3.0259043199998814, + "min": 2.220771276999585, + "rounds": 10, + "stddev": 0.4621819373668309 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.3687198659999922, + "iterations": 1, + "max": 6.344298512000023, + "mean": 2.5350944988001176, + "median": 2.2528185269998175, + "min": 1.4673928570000498, + "rounds": 10, + "stddev": 1.369550412775681 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.8171835169996484, + "iterations": 1, + "max": 28.178666431000238, + "mean": 25.036239308900257, + "median": 24.812968027500574, + "min": 24.174104576999525, + "rounds": 10, + "stddev": 1.1792885207122132 + } + }, + "started": "2021-12-10T13:28:02.554596+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-chatty.json new file mode 100644 index 00000000..8a695a84 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-10T12:40:14.347075+00:00", + "golden": "true", + "id": "run-2021-12-10T10_54_54_920_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 13629.333333333336, + "max": 42619.73333333334, + "mean": 31938.780703226563, + "median": 38332.8, + "min": 1530.9333333333334, + "non_zero_mean": 31938.780703226563, + "non_zero_median": 38332.8, + "percentile25": 26437.26666666667, + "percentile75": 40066.600000000006, + "percentile90": 41113.013333333336, + "percentile99": 42252.27240495078, + "percentile999": 42582.98724049509, + "range": 41088.8, + "samples": 65, + "stdev": 12826.551157809407, + "sum": 2076020.7457097273 + } + }, + "cpu": { + "system": { + "iqr": 0.17133333333333312, + "max": 1.0639999999999996, + "mean": 0.8426256206504842, + "median": 0.997333333333332, + "min": 0.030000000000000186, + "non_zero_mean": 0.8426256206504842, + "non_zero_median": 0.997333333333332, + "percentile25": 0.8540000000000016, + "percentile75": 1.0253333333333348, + "percentile90": 1.0350666666666677, + "percentile99": 1.0567466666666663, + "percentile999": 1.0632746666666664, + "range": 1.0339999999999994, + "samples": 65, + "stdev": 0.303114548770089, + "sum": 54.77066534228147 + }, + "user": { + "iqr": 1.7700000000000005, + "max": 6.560666666666661, + "mean": 5.117466741382017, + "median": 6.327999999999998, + "min": 0.20666666666666816, + "non_zero_mean": 5.117466741382017, + "non_zero_median": 6.327999999999998, + "percentile25": 4.627333333333335, + "percentile75": 6.397333333333336, + "percentile90": 6.439999999999997, + "percentile99": 6.506616883792249, + "percentile999": 6.55526168837922, + "range": 6.353999999999993, + "samples": 65, + "stdev": 2.029336495679382, + "sum": 332.63533818983103 + } + }, + "entropy_available_bits": { + "iqr": 7.866666666666667, + "max": 423.6807742738405, + "mean": 21.73149594737873, + "median": 3.8, + "min": 0.0, + "non_zero_mean": 33.63207706141946, + "non_zero_median": 7.733333333333333, + "percentile25": 0.0, + "percentile75": 7.866666666666667, + "percentile90": 12.906666666666666, + "percentile99": 417.5277454052492, + "percentile999": 423.0654713869814, + "range": 423.6807742738405, + "samples": 65, + "stdev": 79.48479093773368, + "sum": 1412.5472365796172 + }, + "memory": { + "used": { + "iqr": 1229418496.0, + "max": 7403261952.0, + "mean": 6389520856.615385, + "median": 6822506496.0, + "min": 3423158272.0, + "non_zero_mean": 6389520856.615385, + "non_zero_median": 6822506496.0, + "percentile25": 5839507456.0, + "percentile75": 7068925952.0, + "percentile90": 7212157337.6, + "percentile99": 7367767654.4, + "percentile999": 7399712522.240001, + "range": 3980103680.0, + "samples": 65, + "stdev": 931645509.7227057, + "sum": 415318855680.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.006153846153846154, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2, + "non_zero_median": 0.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.18133333333333326, + "percentile999": 0.25813333333333427, + "range": 0.26666666666666666, + "samples": 65, + "stdev": 0.03674816118430078, + "sum": 0.4 + } + }, + "writes_completed": { + "total": { + "iqr": 2586.7333333333336, + "max": 4923.0, + "mean": 3536.239846046299, + "median": 4551.733333333334, + "min": 4.466666666666667, + "non_zero_mean": 3536.239846046299, + "non_zero_median": 4551.733333333334, + "percentile25": 2126.0666666666666, + "percentile75": 4712.8, + "percentile90": 4784.16, + "percentile99": 4898.765333333333, + "percentile999": 4920.576533333334, + "range": 4918.533333333334, + "samples": 65, + "stdev": 1671.23984817948, + "sum": 229855.58999300943 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 199.33333333333326, + "max": 2338.0, + "mean": 1643.3097435897437, + "median": 1702.0666666666666, + "min": 450.4, + "non_zero_mean": 1643.3097435897437, + "non_zero_median": 1702.0666666666666, + "percentile25": 1593.4666666666667, + "percentile75": 1792.8, + "percentile90": 1948.76, + "percentile99": 2211.578666666667, + "percentile999": 2325.357866666668, + "range": 1887.6, + "samples": 65, + "stdev": 322.1710567106413, + "sum": 106815.13333333332 + } + }, + "cpu": { + "system": { + "iqr": 0.03133333333333255, + "max": 0.24200000000000066, + "mean": 0.1435076923076923, + "median": 0.15333333333333363, + "min": 0.022666666666666422, + "non_zero_mean": 0.1435076923076923, + "non_zero_median": 0.15333333333333363, + "percentile25": 0.1333333333333338, + "percentile75": 0.16466666666666635, + "percentile90": 0.17986666666666704, + "percentile99": 0.22322666666666707, + "percentile999": 0.2401226666666675, + "range": 0.21933333333333424, + "samples": 65, + "stdev": 0.04365462253159534, + "sum": 9.328 + }, + "user": { + "iqr": 0.04666666666666616, + "max": 0.5380000000000005, + "mean": 0.22173333333333334, + "median": 0.17733333333333384, + "min": 0.030666666666666245, + "non_zero_mean": 0.22173333333333334, + "non_zero_median": 0.17733333333333384, + "percentile25": 0.15866666666666707, + "percentile75": 0.20533333333333323, + "percentile90": 0.5104000000000003, + "percentile99": 0.537146666666667, + "percentile999": 0.5379146666666671, + "range": 0.5073333333333342, + "samples": 65, + "stdev": 0.13955951538083444, + "sum": 14.412666666666665 + } + }, + "entropy_available_bits": { + "iqr": 1.666666666666667, + "max": 186.46666666666667, + "mean": 9.542564102564102, + "median": 1.8, + "min": 0.8666666666666667, + "non_zero_mean": 9.542564102564102, + "non_zero_median": 1.8, + "percentile25": 1.4, + "percentile75": 3.066666666666667, + "percentile90": 4.693333333333334, + "percentile99": 173.8373333333333, + "percentile999": 185.20373333333347, + "range": 185.6, + "samples": 65, + "stdev": 33.636564258678455, + "sum": 620.2666666666664 + }, + "memory": { + "used": { + "iqr": 23994368.0, + "max": 501047296.0, + "mean": 452461174.15384614, + "median": 461778944.0, + "min": 370814976.0, + "non_zero_mean": 452461174.15384614, + "non_zero_median": 461778944.0, + "percentile25": 450101248.0, + "percentile75": 474095616.0, + "percentile90": 481354547.2, + "percentile99": 491345346.56, + "percentile999": 500077101.0560001, + "range": 130232320.0, + "samples": 65, + "stdev": 32553659.35993257, + "sum": 29409976320.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1736704.0, + "max": 3376742.4, + "mean": 803488.1641025641, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2008720.4102564103, + "non_zero_median": 2158045.8666666667, + "percentile25": 0.0, + "percentile75": 1736704.0, + "percentile90": 2699537.0666666673, + "percentile99": 3357518.506666667, + "percentile999": 3374820.010666667, + "range": 3376742.4, + "samples": 65, + "stdev": 1174476.0291657662, + "sum": 52226730.66666666 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.03063, + "mean": 0.002169476923076923, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0141016, + "non_zero_median": 0.0126125, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.010243600000000002, + "percentile99": 0.025616879999999998, + "percentile999": 0.030128688000000056, + "range": 0.03063, + "samples": 65, + "stdev": 0.006131138528822798, + "sum": 0.14101600000000003 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 0.41629, + "mean": 0.012833723076923077, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.069516, + "non_zero_median": 0.012602, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.008142000000000003, + "percentile99": 0.30174343999999986, + "percentile999": 0.40483534400000126, + "range": 0.41629, + "samples": 65, + "stdev": 0.05995687786820929, + "sum": 0.834192 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 11.8, + "mean": 0.40102564102564103, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8.68888888888889, + "non_zero_median": 7.6, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9.111999999999998, + "percentile999": 11.53120000000003, + "range": 11.8, + "samples": 65, + "stdev": 1.8997986847913764, + "sum": 26.066666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9985.066666666668, + "mean": 460.2041025641026, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9971.08888888889, + "non_zero_median": 9981.4, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9982.72, + "percentile999": 9984.832, + "range": 9985.066666666668, + "samples": 65, + "stdev": 2108.400328509907, + "sum": 29913.266666666663 + }, + "pg_stat_database_blks_hit": { + "iqr": 2716.0, + "max": 37080.666666666664, + "mean": 21814.10153846154, + "median": 22965.466666666667, + "min": 6103.6, + "non_zero_mean": 21814.10153846154, + "non_zero_median": 22965.466666666667, + "percentile25": 21089.133333333335, + "percentile75": 23805.133333333335, + "percentile90": 25009.053333333333, + "percentile99": 36442.928, + "percentile999": 37016.8928, + "range": 30977.066666666666, + "samples": 65, + "stdev": 5911.404434240128, + "sum": 1417916.5999999999 + }, + "pg_stat_database_blks_read": { + "iqr": 68.33333333333333, + "max": 150.6, + "mean": 100.3076923076923, + "median": 130.6, + "min": 0.0, + "non_zero_mean": 101.875, + "non_zero_median": 131.0, + "percentile25": 68.33333333333333, + "percentile75": 136.66666666666666, + "percentile90": 142.8, + "percentile99": 149.14933333333335, + "percentile999": 150.45493333333334, + "range": 150.6, + "samples": 65, + "stdev": 50.81992540702976, + "sum": 6519.999999999999 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 65, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 68.33333333333333, + "max": 150.6, + "mean": 100.3076923076923, + "median": 130.6, + "min": 0.0, + "non_zero_mean": 101.875, + "non_zero_median": 131.0, + "percentile25": 68.33333333333333, + "percentile75": 136.66666666666666, + "percentile90": 142.8, + "percentile99": 149.14933333333335, + "percentile999": 150.45493333333334, + "range": 150.6, + "samples": 65, + "stdev": 50.81992540702976, + "sum": 6519.999999999999 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4, + "mean": 0.19897435897435897, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8083333333333333, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9733333333333334, + "percentile99": 1.3573333333333333, + "percentile999": 1.3957333333333337, + "range": 1.4, + "samples": 65, + "stdev": 0.3958977680349344, + "sum": 12.933333333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 3033.4666666666662, + "max": 25284.333333333332, + "mean": 7220.1456410256405, + "median": 5150.866666666667, + "min": 3634.5333333333333, + "non_zero_mean": 7220.1456410256405, + "non_zero_median": 5150.866666666667, + "percentile25": 4882.6, + "percentile75": 7916.066666666667, + "percentile90": 13648.533333333333, + "percentile99": 21240.04533333333, + "percentile999": 24879.904533333378, + "range": 21649.8, + "samples": 65, + "stdev": 4363.505448077464, + "sum": 469309.46666666673 + }, + "pg_stat_database_tup_inserted": { + "iqr": 366.2666666666667, + "max": 814.9333333333333, + "mean": 533.9723076923077, + "median": 698.5333333333333, + "min": 0.0, + "non_zero_mean": 542.315625, + "non_zero_median": 700.2666666666667, + "percentile25": 358.93333333333334, + "percentile75": 725.2, + "percentile90": 758.6, + "percentile99": 810.4533333333333, + "percentile999": 814.4853333333333, + "range": 814.9333333333333, + "samples": 65, + "stdev": 269.16855466115663, + "sum": 34708.200000000004 + }, + "pg_stat_database_tup_returned": { + "iqr": 2989.333333333334, + "max": 29386.466666666667, + "mean": 9143.969230769231, + "median": 6506.733333333334, + "min": 4567.866666666667, + "non_zero_mean": 9143.969230769231, + "non_zero_median": 6506.733333333334, + "percentile25": 6094.0, + "percentile75": 9083.333333333334, + "percentile90": 18932.08, + "percentile99": 26751.842666666664, + "percentile999": 29123.004266666696, + "range": 24818.6, + "samples": 65, + "stdev": 5658.612002478524, + "sum": 594357.9999999998 + }, + "pg_stat_database_tup_updated": { + "iqr": 5.8, + "max": 135.8, + "mean": 11.448205128205128, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 14.040251572327044, + "non_zero_median": 1.4666666666666666, + "percentile25": 0.2, + "percentile75": 6.0, + "percentile90": 37.80000000000001, + "percentile99": 109.21866666666665, + "percentile999": 133.14186666666697, + "range": 135.8, + "samples": 65, + "stdev": 26.274191254316012, + "sum": 744.1333333333333 + }, + "pg_stat_database_xact_commit": { + "iqr": 12.733333333333334, + "max": 152.4, + "mean": 58.98358974358975, + "median": 50.8, + "min": 9.2, + "non_zero_mean": 58.98358974358975, + "non_zero_median": 50.8, + "percentile25": 48.333333333333336, + "percentile75": 61.06666666666667, + "percentile90": 87.13333333333334, + "percentile99": 139.68533333333332, + "percentile999": 151.12853333333348, + "range": 143.20000000000002, + "samples": 65, + "stdev": 25.960250145305565, + "sum": 3833.933333333333 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 65, + "stdev": 0.0, + "sum": 4.3333333333333375 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.2, + "mean": 0.03282051282051282, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.711111111111111, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.8586666666666664, + "percentile999": 1.1658666666666704, + "range": 1.2, + "samples": 65, + "stdev": 0.17164363798736748, + "sum": 2.1333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 21.39999999999999, + "max": 148.46666666666667, + "mean": 78.39179487179487, + "median": 81.13333333333333, + "min": 5.866666666666667, + "non_zero_mean": 78.39179487179487, + "non_zero_median": 81.13333333333333, + "percentile25": 70.66666666666667, + "percentile75": 92.06666666666666, + "percentile90": 106.18666666666668, + "percentile99": 134.55733333333333, + "percentile999": 147.0757333333335, + "range": 142.6, + "samples": 65, + "stdev": 25.615724629426325, + "sum": 5095.466666666665 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,8,9,10,11,14,18,20,19,21,22,23,25,24,26,27,28,29,31,30,32,33,35,34,36,37,38,39,40,41,42,44,45,43,46,47,48,49,51,52,54,50,53,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-10T12:21:26.788194Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T12:21:27.690760Z 2021-12-10T12:21:27.668926Z 2021-12-10T12:21:27.321176Z 2021-12-10T12:21:26.959931Z 2021-12-10T12:21:26.951623Z 2021-12-10T12:21:26.901377Z 2021-12-10T12:21:26.788194Z 2021-12-10T12:21:28.685425Z 2021-12-10T12:21:27.743075Z 2021-12-10T12:21:27.742824Z 2021-12-10T12:25:44.334690Z 2021-12-10T12:25:44.207983Z 2021-12-10T12:25:43.853993Z 2021-12-10T12:25:43.852675Z 2021-12-10T12:25:43.846547Z 2021-12-10T12:25:43.811837Z 2021-12-10T12:25:43.769384Z 2021-12-10T12:25:43.748913Z 2021-12-10T12:25:43.629926Z 2021-12-10T12:25:43.619329Z 2021-12-10T12:28:51.380133Z 2021-12-10T12:28:50.513034Z 2021-12-10T12:28:49.998195Z 2021-12-10T12:28:49.956720Z 2021-12-10T12:28:49.954315Z 2021-12-10T12:28:49.844411Z 2021-12-10T12:28:49.844124Z 2021-12-10T12:28:49.831919Z 2021-12-10T12:28:49.828977Z 2021-12-10T12:28:49.824347Z 2021-12-10T12:31:51.405128Z 2021-12-10T12:31:51.342782Z 2021-12-10T12:31:50.379327Z 2021-12-10T12:31:50.362975Z 2021-12-10T12:31:50.337651Z 2021-12-10T12:31:50.324833Z 2021-12-10T12:31:50.284544Z 2021-12-10T12:31:50.273529Z 2021-12-10T12:31:50.229336Z 2021-12-10T12:31:50.225524Z 2021-12-10T12:34:51.934585Z 2021-12-10T12:34:51.914666Z 2021-12-10T12:34:51.478388Z 2021-12-10T12:34:51.417877Z 2021-12-10T12:34:51.338740Z 2021-12-10T12:34:51.335480Z 2021-12-10T12:34:51.335249Z 2021-12-10T12:34:51.326110Z 2021-12-10T12:34:51.322290Z 2021-12-10T12:34:51.301005Z", + "created_diff": 1.1963124, + "ended": "2021-12-10T12:37:29.475006Z", + "error_job_ids": " []", + "event_first": "2021-12-10T12:21:39.220091Z 2021-12-10T12:23:08.293493Z 2021-12-10T12:21:40.906699Z 2021-12-10T12:22:13.282797Z 2021-12-10T12:21:32.916765Z 2021-12-10T12:22:50.827578Z 2021-12-10T12:21:32.834702Z 2021-12-10T12:22:24.092765Z 2021-12-10T12:21:40.316370Z 2021-12-10T12:22:35.098518Z 2021-12-10T12:25:58.097763Z 2021-12-10T12:26:03.434384Z 2021-12-10T12:25:58.350829Z 2021-12-10T12:25:59.390240Z 2021-12-10T12:25:58.349762Z 2021-12-10T12:25:49.649373Z 2021-12-10T12:25:49.324031Z 2021-12-10T12:25:49.643347Z 2021-12-10T12:25:48.824905Z 2021-12-10T12:25:49.134509Z 2021-12-10T12:29:03.196734Z 2021-12-10T12:29:02.661683Z 2021-12-10T12:29:02.260507Z 2021-12-10T12:29:02.661683Z 2021-12-10T12:28:59.478697Z 2021-12-10T12:28:57.153541Z 2021-12-10T12:28:56.414619Z 2021-12-10T12:28:57.657818Z 2021-12-10T12:28:56.677426Z 2021-12-10T12:28:56.119504Z 2021-12-10T12:32:05.589904Z 2021-12-10T12:32:05.341449Z 2021-12-10T12:32:03.188310Z 2021-12-10T12:32:00.239541Z 2021-12-10T12:31:59.327965Z 2021-12-10T12:31:58.241322Z 2021-12-10T12:31:57.272770Z 2021-12-10T12:31:57.744011Z 2021-12-10T12:31:56.772403Z 2021-12-10T12:31:58.500674Z 2021-12-10T12:35:06.358703Z 2021-12-10T12:35:07.898420Z 2021-12-10T12:35:03.873761Z 2021-12-10T12:35:02.949609Z 2021-12-10T12:34:58.470969Z 2021-12-10T12:35:00.353841Z 2021-12-10T12:34:58.988768Z 2021-12-10T12:34:58.152998Z 2021-12-10T12:34:58.004700Z 2021-12-10T12:34:58.646417Z", + "event_last": "2021-12-10T12:24:11.870471Z 2021-12-10T12:25:18.818797Z 2021-12-10T12:24:15.203137Z 2021-12-10T12:24:42.293953Z 2021-12-10T12:24:05.599069Z 2021-12-10T12:25:05.932031Z 2021-12-10T12:24:05.142835Z 2021-12-10T12:24:50.289100Z 2021-12-10T12:24:14.424084Z 2021-12-10T12:24:55.742655Z 2021-12-10T12:28:20.020705Z 2021-12-10T12:28:22.916934Z 2021-12-10T12:28:20.408398Z 2021-12-10T12:28:21.725843Z 2021-12-10T12:28:20.717934Z 2021-12-10T12:28:14.507527Z 2021-12-10T12:28:12.254787Z 2021-12-10T12:28:14.634298Z 2021-12-10T12:28:12.185545Z 2021-12-10T12:28:15.299347Z 2021-12-10T12:31:26.413338Z 2021-12-10T12:31:26.605562Z 2021-12-10T12:31:24.968629Z 2021-12-10T12:31:26.351219Z 2021-12-10T12:31:22.176639Z 2021-12-10T12:31:23.071324Z 2021-12-10T12:31:21.776954Z 2021-12-10T12:31:24.272364Z 2021-12-10T12:31:22.151658Z 2021-12-10T12:31:21.796468Z 2021-12-10T12:34:28.607135Z 2021-12-10T12:34:30.740374Z 2021-12-10T12:34:26.671298Z 2021-12-10T12:34:26.464956Z 2021-12-10T12:34:24.288544Z 2021-12-10T12:34:23.798336Z 2021-12-10T12:34:21.717869Z 2021-12-10T12:34:25.018899Z 2021-12-10T12:34:22.671982Z 2021-12-10T12:34:24.711688Z 2021-12-10T12:37:27.579939Z 2021-12-10T12:37:29.475006Z 2021-12-10T12:37:26.106788Z 2021-12-10T12:37:24.773001Z 2021-12-10T12:37:19.033960Z 2021-12-10T12:37:22.059992Z 2021-12-10T12:37:22.962626Z 2021-12-10T12:37:20.503743Z 2021-12-10T12:37:21.692095Z 2021-12-10T12:37:21.557778Z", + "failed_job_ids": " []", + "finished": "2021-12-10T12:24:15.516067Z 2021-12-10T12:25:20.931789Z 2021-12-10T12:24:19.212685Z 2021-12-10T12:24:46.484199Z 2021-12-10T12:24:09.602200Z 2021-12-10T12:25:09.886814Z 2021-12-10T12:24:09.412410Z 2021-12-10T12:24:53.655803Z 2021-12-10T12:24:18.669701Z 2021-12-10T12:25:00.072998Z 2021-12-10T12:28:23.882680Z 2021-12-10T12:28:26.787235Z 2021-12-10T12:28:23.746199Z 2021-12-10T12:28:25.314379Z 2021-12-10T12:28:24.365882Z 2021-12-10T12:28:18.555461Z 2021-12-10T12:28:16.175269Z 2021-12-10T12:28:18.301790Z 2021-12-10T12:28:16.110636Z 2021-12-10T12:28:19.301446Z 2021-12-10T12:31:30.304477Z 2021-12-10T12:31:30.675810Z 2021-12-10T12:31:28.540789Z 2021-12-10T12:31:30.228628Z 2021-12-10T12:31:26.326172Z 2021-12-10T12:31:27.235526Z 2021-12-10T12:31:25.946489Z 2021-12-10T12:31:27.953970Z 2021-12-10T12:31:25.869949Z 2021-12-10T12:31:25.737503Z 2021-12-10T12:34:32.353937Z 2021-12-10T12:34:32.055389Z 2021-12-10T12:34:30.102453Z 2021-12-10T12:34:30.663374Z 2021-12-10T12:34:28.573270Z 2021-12-10T12:34:28.151305Z 2021-12-10T12:34:25.496976Z 2021-12-10T12:34:28.972288Z 2021-12-10T12:34:27.085667Z 2021-12-10T12:34:28.528684Z 2021-12-10T12:37:31.495496Z 2021-12-10T12:37:31.645198Z 2021-12-10T12:37:30.148797Z 2021-12-10T12:37:28.351100Z 2021-12-10T12:37:23.356064Z 2021-12-10T12:37:25.839766Z 2021-12-10T12:37:26.563070Z 2021-12-10T12:37:24.272838Z 2021-12-10T12:37:25.991510Z 2021-12-10T12:37:25.456108Z", + "finished_diff": 20.456076, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 231.789963, + "mean": 161.95570155999997, + "min": 150.64288 + }, + "jobs_events_duration": { + "max": 154.296438, + "mean": 144.35639892, + "min": 130.525304 + }, + "jobs_events_lag": { + "max": -1.315015, + "mean": -3.79801264, + "min": -4.413685 + }, + "jobs_waiting": { + "max": 18.410779, + "mean": 2.1629677200000006, + "min": 0.793157 + }, + "modified": "2021-12-10T12:21:28.764146Z 2021-12-10T12:21:28.722388Z 2021-12-10T12:21:28.686011Z 2021-12-10T12:21:27.384805Z 2021-12-10T12:21:27.337788Z 2021-12-10T12:21:27.305431Z 2021-12-10T12:21:27.274486Z 2021-12-10T12:21:46.815509Z 2021-12-10T12:21:28.868342Z 2021-12-10T12:21:28.829182Z 2021-12-10T12:25:48.684112Z 2021-12-10T12:25:48.651309Z 2021-12-10T12:25:48.618188Z 2021-12-10T12:25:48.585757Z 2021-12-10T12:25:48.552926Z 2021-12-10T12:25:44.265071Z 2021-12-10T12:25:44.233809Z 2021-12-10T12:25:44.203529Z 2021-12-10T12:25:44.159658Z 2021-12-10T12:25:44.122881Z 2021-12-10T12:28:52.534534Z 2021-12-10T12:28:52.496575Z 2021-12-10T12:28:52.455518Z 2021-12-10T12:28:52.412980Z 2021-12-10T12:28:50.652731Z 2021-12-10T12:28:50.619468Z 2021-12-10T12:28:50.585773Z 2021-12-10T12:28:50.553357Z 2021-12-10T12:28:50.520747Z 2021-12-10T12:28:50.487031Z 2021-12-10T12:31:52.613398Z 2021-12-10T12:31:52.543020Z 2021-12-10T12:31:51.019845Z 2021-12-10T12:31:50.990199Z 2021-12-10T12:31:50.960356Z 2021-12-10T12:31:50.930614Z 2021-12-10T12:31:50.900612Z 2021-12-10T12:31:50.870743Z 2021-12-10T12:31:50.839596Z 2021-12-10T12:31:50.808265Z 2021-12-10T12:34:54.125915Z 2021-12-10T12:34:54.057650Z 2021-12-10T12:34:52.111144Z 2021-12-10T12:34:52.078560Z 2021-12-10T12:34:52.045909Z 2021-12-10T12:34:52.009995Z 2021-12-10T12:34:51.976181Z 2021-12-10T12:34:51.943902Z 2021-12-10T12:34:51.911580Z 2021-12-10T12:34:51.875564Z", + "modified_diff": 6.0410482000000005, + "started": "2021-12-10T12:21:29.207342Z 2021-12-10T12:21:29.141826Z 2021-12-10T12:21:29.061364Z 2021-12-10T12:21:27.940552Z 2021-12-10T12:21:27.750553Z 2021-12-10T12:21:27.728229Z 2021-12-10T12:21:27.581351Z 2021-12-10T12:21:47.096204Z 2021-12-10T12:21:29.429891Z 2021-12-10T12:21:29.316288Z 2021-12-10T12:25:49.695359Z 2021-12-10T12:25:49.604011Z 2021-12-10T12:25:49.335213Z 2021-12-10T12:25:49.187844Z 2021-12-10T12:25:49.006533Z 2021-12-10T12:25:44.757235Z 2021-12-10T12:25:44.673604Z 2021-12-10T12:25:44.607562Z 2021-12-10T12:25:44.529334Z 2021-12-10T12:25:44.451631Z 2021-12-10T12:28:53.172594Z 2021-12-10T12:28:53.114786Z 2021-12-10T12:28:52.904936Z 2021-12-10T12:28:52.802156Z 2021-12-10T12:28:51.357919Z 2021-12-10T12:28:51.258247Z 2021-12-10T12:28:51.177990Z 2021-12-10T12:28:51.083265Z 2021-12-10T12:28:51.009657Z 2021-12-10T12:28:50.916550Z 2021-12-10T12:31:53.117966Z 2021-12-10T12:31:52.926634Z 2021-12-10T12:31:52.060018Z 2021-12-10T12:31:52.018314Z 2021-12-10T12:31:51.712765Z 2021-12-10T12:31:51.655556Z 2021-12-10T12:31:51.544805Z 2021-12-10T12:31:51.446483Z 2021-12-10T12:31:51.361769Z 2021-12-10T12:31:51.270705Z 2021-12-10T12:34:54.660838Z 2021-12-10T12:34:54.548579Z 2021-12-10T12:34:52.985975Z 2021-12-10T12:34:52.882829Z 2021-12-10T12:34:52.713184Z 2021-12-10T12:34:52.633319Z 2021-12-10T12:34:52.558626Z 2021-12-10T12:34:52.453259Z 2021-12-10T12:34:52.366703Z 2021-12-10T12:34:52.304815Z", + "started_diff": 6.243581799999999, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-10T12:40:11.346040+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-launching.json new file mode 100644 index 00000000..5eddd01f --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T10_54_54_920_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-10T13:27:08.141970+00:00", + "golden": "true", + "id": "run-2021-12-10T10_54_54_920_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17824.033333333333, + "max": 29401.333333333336, + "mean": 9782.359090909447, + "median": 4575.0, + "min": 1350.3333333333333, + "non_zero_mean": 9782.359090909447, + "non_zero_median": 4575.0, + "percentile25": 1755.6166666666668, + "percentile75": 19579.65, + "percentile90": 23786.573333333345, + "percentile99": 28127.800666666655, + "percentile999": 29273.980066666678, + "range": 28051.000000000004, + "samples": 80, + "stdev": 9492.229850663747, + "sum": 782588.7272727558 + } + }, + "cpu": { + "system": { + "iqr": 1.3523333333333363, + "max": 1.8006666666666662, + "mean": 0.6481082302069961, + "median": 0.23866666666666847, + "min": 0.027333333333327423, + "non_zero_mean": 0.6481082302069961, + "non_zero_median": 0.23866666666666847, + "percentile25": 0.039333333333331194, + "percentile75": 1.3916666666666675, + "percentile90": 1.608066666666667, + "percentile99": 1.793293333333332, + "percentile999": 1.7999293333333328, + "range": 1.7733333333333388, + "samples": 80, + "stdev": 0.6897412592500693, + "sum": 51.84865841655969 + }, + "user": { + "iqr": 5.808666666666645, + "max": 6.558000000000001, + "mean": 2.7811499301926514, + "median": 1.0006666666666622, + "min": 0.08600000000000516, + "non_zero_mean": 2.7811499301926514, + "non_zero_median": 1.0006666666666622, + "percentile25": 0.31566666666668086, + "percentile75": 6.124333333333326, + "percentile90": 6.348933333333338, + "percentile99": 6.553259999999975, + "percentile999": 6.557525999999998, + "range": 6.471999999999996, + "samples": 80, + "stdev": 2.768409079916421, + "sum": 222.4919944154121 + } + }, + "entropy_available_bits": { + "iqr": 3.966666666666666, + "max": 214.13333333333335, + "mean": 22.912500167788917, + "median": 0.9333466684446815, + "min": 0.0, + "non_zero_mean": 24.440000178974845, + "non_zero_median": 1.0, + "percentile25": 0.6666666666666667, + "percentile75": 4.633333333333333, + "percentile90": 26.720000000001733, + "percentile99": 213.34333333333333, + "percentile999": 214.05433333333335, + "range": 214.13333333333335, + "samples": 80, + "stdev": 63.399723168436005, + "sum": 1833.0000134231136 + }, + "memory": { + "used": { + "iqr": 6124898304.0, + "max": 17068032000.0, + "mean": 8685886771.2, + "median": 6551001088.0, + "min": 5045821440.0, + "non_zero_mean": 8685886771.2, + "non_zero_median": 6551001088.0, + "percentile25": 6077782016.0, + "percentile75": 12202680320.0, + "percentile90": 15334684262.4, + "percentile99": 17022665523.199999, + "percentile999": 17063495352.32, + "range": 12022210560.0, + "samples": 80, + "stdev": 3818184576.7338753, + "sum": 694870941696.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.7333333333333334, + "mean": 0.073331555792561, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.173304892680976, + "non_zero_median": 1.0666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.7333333333333334, + "percentile999": 1.7333333333333334, + "range": 1.7333333333333334, + "samples": 80, + "stdev": 0.3167287471579284, + "sum": 5.866524463404879 + } + }, + "writes_completed": { + "total": { + "iqr": 913.3333333333333, + "max": 1691.9719389455443, + "mean": 453.3609437821775, + "median": 89.0, + "min": 0.5333333333333333, + "non_zero_mean": 453.3609437821775, + "non_zero_median": 89.0, + "percentile25": 3.7, + "percentile75": 917.0333333333333, + "percentile90": 1463.410398079616, + "percentile99": 1640.3281071785639, + "percentile999": 1686.8075557688467, + "range": 1691.438605612211, + "samples": 80, + "stdev": 596.5551901805852, + "sum": 36268.87550257421 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2161.0499999999997, + "max": 3969.6, + "mean": 1685.1641666666667, + "median": 869.8, + "min": 492.6666666666667, + "non_zero_mean": 1685.1641666666667, + "non_zero_median": 869.8, + "percentile25": 662.7166666666667, + "percentile75": 2823.7666666666664, + "percentile90": 3302.38, + "percentile99": 3633.3233333333305, + "percentile999": 3935.972333333336, + "range": 3476.9333333333334, + "samples": 80, + "stdev": 1142.2573422090666, + "sum": 134813.13333333336 + } + }, + "cpu": { + "system": { + "iqr": 0.20383333333333245, + "max": 0.40733333333333327, + "mean": 0.127375, + "median": 0.043333333333333474, + "min": 0.013333333333333523, + "non_zero_mean": 0.127375, + "non_zero_median": 0.043333333333333474, + "percentile25": 0.02400000000000091, + "percentile75": 0.22783333333333336, + "percentile90": 0.3014666666666665, + "percentile99": 0.3952199999999996, + "percentile999": 0.40612200000000004, + "range": 0.39399999999999974, + "samples": 80, + "stdev": 0.11980830181689217, + "sum": 10.19 + }, + "user": { + "iqr": 0.3036666666666655, + "max": 0.8660000000000025, + "mean": 0.19524166666666667, + "median": 0.056666666666665824, + "min": 0.012000000000002349, + "non_zero_mean": 0.19524166666666667, + "non_zero_median": 0.056666666666665824, + "percentile25": 0.0373333333333342, + "percentile75": 0.3409999999999997, + "percentile90": 0.44680000000000086, + "percentile99": 0.69694, + "percentile999": 0.8490940000000038, + "range": 0.8540000000000002, + "samples": 80, + "stdev": 0.19824426943970455, + "sum": 15.619333333333332 + } + }, + "entropy_available_bits": { + "iqr": 2.583333333333333, + "max": 212.33333333333334, + "mean": 11.548333333333334, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 16.497619047619047, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 2.583333333333333, + "percentile90": 4.360000000000002, + "percentile99": 211.70133333333334, + "percentile999": 212.27013333333335, + "range": 212.33333333333334, + "samples": 80, + "stdev": 45.83418162621195, + "sum": 923.8666666666667 + }, + "memory": { + "used": { + "iqr": 414152704.0, + "max": 1125289984.0, + "mean": 581923635.2, + "median": 405813248.0, + "min": 368312320.0, + "non_zero_mean": 581923635.2, + "non_zero_median": 405813248.0, + "percentile25": 381198336.0, + "percentile75": 795351040.0, + "percentile90": 1027741696.0, + "percentile99": 1117698703.36, + "percentile999": 1124530855.936, + "range": 756977664.0, + "samples": 80, + "stdev": 263519406.5986594, + "sum": 46553890816.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 3959.4666666666667, + "max": 55548372.2, + "mean": 709967.2391666668, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2704637.1015873016, + "non_zero_median": 52974.933333333334, + "percentile25": 0.0, + "percentile75": 3959.4666666666667, + "percentile90": 63624.533333333355, + "percentile99": 11803652.113999655, + "percentile999": 51173900.191400364, + "range": 55548372.2, + "samples": 80, + "stdev": 6208850.322419024, + "sum": 56797379.13333333 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 1.511944, + "mean": 0.0427516, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.18000673684210527, + "non_zero_median": 0.030656, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.14484240000000004, + "percentile99": 0.6962223899999935, + "percentile999": 1.4303718390000066, + "range": 1.511944, + "samples": 80, + "stdev": 0.1818957057200845, + "sum": 3.4201279999999996 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.18339975, + "max": 6.321843, + "mean": 0.4099910875, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.9939177878787879, + "non_zero_median": 0.357441, + "percentile25": 0.0, + "percentile75": 0.18339975, + "percentile90": 1.2335546000000013, + "percentile99": 5.15874653999999, + "percentile999": 6.20553335400001, + "range": 6.321843, + "samples": 80, + "stdev": 1.056440363556923, + "sum": 32.79928700000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.333333333333333, + "mean": 0.21, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.36, + "non_zero_median": 3.4, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.227999999999999, + "percentile999": 4.322800000000001, + "range": 4.333333333333333, + "samples": 80, + "stdev": 0.8442808339541283, + "sum": 16.8 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9996.466666666667, + "mean": 570.22, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9123.52, + "non_zero_median": 9982.933333333332, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9986.302, + "percentile999": 9995.450200000001, + "range": 9996.466666666667, + "samples": 80, + "stdev": 2244.8342762334287, + "sum": 45617.6 + }, + "pg_stat_database_blks_hit": { + "iqr": 28307.066666666662, + "max": 110844.86666666667, + "mean": 21060.7125, + "median": 8779.433333333334, + "min": 1403.8666666666666, + "non_zero_mean": 21060.7125, + "non_zero_median": 8779.433333333334, + "percentile25": 6999.483333333334, + "percentile75": 35306.549999999996, + "percentile90": 51803.920000000006, + "percentile99": 80511.6053333331, + "percentile999": 107811.54053333358, + "range": 109441.0, + "samples": 80, + "stdev": 21461.691002585856, + "sum": 1684857.0000000005 + }, + "pg_stat_database_blks_read": { + "iqr": 2.8833333333333333, + "max": 14.733333333333333, + "mean": 2.0441666666666665, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.3374149659863948, + "non_zero_median": 2.066666666666667, + "percentile25": 0.0, + "percentile75": 2.8833333333333333, + "percentile90": 5.813333333333346, + "percentile99": 14.101333333333328, + "percentile999": 14.670133333333338, + "range": 14.733333333333333, + "samples": 80, + "stdev": 3.3228856310469492, + "sum": 163.5333333333333 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 80, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 2.8833333333333333, + "max": 14.733333333333333, + "mean": 2.0441666666666665, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.3374149659863948, + "non_zero_median": 2.066666666666667, + "percentile25": 0.0, + "percentile75": 2.8833333333333333, + "percentile90": 5.813333333333346, + "percentile99": 14.101333333333328, + "percentile999": 14.670133333333338, + "range": 14.733333333333333, + "samples": 80, + "stdev": 3.3228856310469492, + "sum": 163.5333333333333 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.3333333333333333, + "mean": 0.16666666666666666, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.9523809523809523, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.3333333333333333, + "percentile999": 1.3333333333333333, + "range": 1.3333333333333333, + "samples": 80, + "stdev": 0.389453076782648, + "sum": 13.333333333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 16080.883333333331, + "max": 56120.13333333333, + "mean": 11220.010833333334, + "median": 4592.366666666667, + "min": 701.4666666666667, + "non_zero_mean": 11220.010833333334, + "non_zero_median": 4592.366666666667, + "percentile25": 3627.483333333333, + "percentile75": 19708.366666666665, + "percentile90": 26499.40666666667, + "percentile99": 40725.508666666545, + "percentile999": 54580.67086666679, + "range": 55418.666666666664, + "samples": 80, + "stdev": 11186.515406170194, + "sum": 897600.8666666667 + }, + "pg_stat_database_tup_inserted": { + "iqr": 23.85, + "max": 114.8, + "mean": 16.105833333333333, + "median": 4.7, + "min": 0.0, + "non_zero_mean": 25.769333333333332, + "non_zero_median": 18.299999999999997, + "percentile25": 0.0, + "percentile75": 23.85, + "percentile90": 46.13333333333335, + "percentile99": 103.42399999999991, + "percentile999": 113.66240000000009, + "range": 114.8, + "samples": 80, + "stdev": 24.51954229364269, + "sum": 1288.4666666666665 + }, + "pg_stat_database_tup_returned": { + "iqr": 22430.15, + "max": 83206.46666666666, + "mean": 17816.0575, + "median": 7553.866666666667, + "min": 1623.4, + "non_zero_mean": 17816.0575, + "non_zero_median": 7553.866666666667, + "percentile25": 6224.0666666666675, + "percentile75": 28654.216666666667, + "percentile90": 43872.613333333335, + "percentile99": 67870.72333333321, + "percentile999": 81672.89233333345, + "range": 81583.06666666667, + "samples": 80, + "stdev": 17438.79985220645, + "sum": 1425284.6 + }, + "pg_stat_database_tup_updated": { + "iqr": 30.233333333333334, + "max": 73.13333333333334, + "mean": 16.323333333333334, + "median": 6.166666666666666, + "min": 0.0, + "non_zero_mean": 18.137037037037036, + "non_zero_median": 9.366666666666667, + "percentile25": 0.2, + "percentile75": 30.433333333333334, + "percentile90": 46.02666666666667, + "percentile99": 73.028, + "percentile999": 73.12280000000001, + "range": 73.13333333333334, + "samples": 80, + "stdev": 19.46114047912511, + "sum": 1305.8666666666668 + }, + "pg_stat_database_xact_commit": { + "iqr": 264.59999999999997, + "max": 681.2, + "mean": 149.19916666666666, + "median": 36.86666666666667, + "min": 2.6666666666666665, + "non_zero_mean": 149.19916666666666, + "non_zero_median": 36.86666666666667, + "percentile25": 9.2, + "percentile75": 273.79999999999995, + "percentile90": 394.76, + "percentile99": 547.7426666666656, + "percentile999": 667.8542666666679, + "range": 678.5333333333334, + "samples": 80, + "stdev": 166.214992187795, + "sum": 11935.933333333336 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 80, + "stdev": 0.0, + "sum": 5.333333333333334 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.006666666666666666, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.26666666666666666, + "percentile999": 0.26666666666666666, + "range": 0.26666666666666666, + "samples": 80, + "stdev": 0.041895993383656335, + "sum": 0.5333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 85.73333333333333, + "max": 166.6, + "mean": 56.718333333333334, + "median": 28.666666666666664, + "min": 1.4666666666666666, + "non_zero_mean": 56.718333333333334, + "non_zero_median": 28.666666666666664, + "percentile25": 11.950000000000001, + "percentile75": 97.68333333333334, + "percentile90": 145.64666666666668, + "percentile99": 161.1753333333333, + "percentile999": 166.05753333333337, + "range": 165.13333333333333, + "samples": 80, + "stdev": 53.37340053499945, + "sum": 4537.466666666668 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "62,60,75,63,68,65,72,61,67,66,78,74,77,71,87,79,70,73,80,89,81,84,82,83,85,86,88,91,92,94,93,90,95,96,99,101,100,98,103,104,106,105,109,110,111,114,116,119,113,115,112,117,120,118,122,123,121,124,128,126,127,130,132,131,133,129,136,137,125,138,135,139,144,143,140,149,142,145,147,151,148,134,155,154,156,157,158,159,160,161,164,167,165,166,163,162,170,168,171,169,172,173,174,175,177,178,179,176,184,185,180,182,186,183,181,187,195,193,198,190,200,188,189,191,194,197,196,203,201,199,192,208,204,205,207,202,215,214,209,216,210,217,218,211,219,212,213,220,221,222,206,223,225,228,226,233,231,229,232,227,230,237,240,235,241,234,236,238,246,245,244,224,239,243,247,253,249,250,256,251,254,248,252,257,255,258,262,242,263,266,260,261,259,264,265,267,270,268,269,271,272,275,276,274,273,278,282,280,283,279,285,277,286,292,287,284,290,295,294,281,293,289,297,291,302,288,303,300,301,306,307,296,298,299,304,305,309,308,313,310,311,312,314,315,317,316,323,321,318,324,322,320,319,330,325,328,329,332,335,331,336,326,333,338,334,341,343,344,340,350,346,337,339,327,342,349,347,348,345,357,351,353,352,355,359,358,362,356,360,361,364,366,354,363,367,368,369,365,371,370,372,374,378,375,373,376,385,386,383,381,387,384,393,388,377,380,379,382,391,390,395,389,405,396,397,392,394,400,399,404,398,401,402,403,406,407,409,410,411,408,414,412,416,413,417,415,419,422,421,420,423,424,428,426,418,427,429,425,432,436,433,430,431,434,435,438,437,442,444,440,439,441,448,446,445,449,458,443,453,447,451,455,454,462,456,459,457,463,460,465,464,452,466,450,461,468,467,470,469,471,472,473,475,479,478,476,481,474,485,477,482,489,487,486,484,488,501,480,490,483,492,493,499,498,496,503,494,500,497,495,491,505,504,502,510,507,509,508,511,514,517,506,519,516,515,523,513,522,520,518,524,512,527,529,521,525,535,526,533,528,534,532,536,540,531,537,538,541,543,547,546,542,544,539,549,545,550,552,551,554,548,555,556,557,558,559,560,553,561,564,563,562,567,530,568,566,569,570,565,571", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-10T12:42:03.578292Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T12:42:09.040556Z 2021-12-10T12:42:08.995361Z 2021-12-10T12:42:08.766995Z 2021-12-10T12:42:08.554952Z 2021-12-10T12:42:08.354802Z 2021-12-10T12:42:08.339296Z 2021-12-10T12:42:08.249737Z 2021-12-10T12:42:08.215187Z 2021-12-10T12:42:08.170492Z 2021-12-10T12:42:07.773568Z 2021-12-10T12:42:07.677333Z 2021-12-10T12:42:07.523846Z 2021-12-10T12:42:07.490511Z 2021-12-10T12:42:07.398123Z 2021-12-10T12:42:07.213862Z 2021-12-10T12:42:07.087994Z 2021-12-10T12:42:06.988760Z 2021-12-10T12:42:06.988752Z 2021-12-10T12:42:06.824027Z 2021-12-10T12:42:06.691979Z 2021-12-10T12:42:06.631059Z 2021-12-10T12:42:06.613187Z 2021-12-10T12:42:06.195080Z 2021-12-10T12:42:06.084444Z 2021-12-10T12:42:06.079477Z 2021-12-10T12:42:06.033017Z 2021-12-10T12:42:05.894895Z 2021-12-10T12:42:05.781023Z 2021-12-10T12:42:05.716233Z 2021-12-10T12:42:05.606370Z 2021-12-10T12:42:05.384720Z 2021-12-10T12:42:05.257255Z 2021-12-10T12:42:04.874808Z 2021-12-10T12:42:04.489766Z 2021-12-10T12:42:04.095226Z 2021-12-10T12:42:03.578292Z 2021-12-10T12:42:29.737639Z 2021-12-10T12:42:28.227266Z 2021-12-10T12:42:28.115291Z 2021-12-10T12:42:26.834240Z 2021-12-10T12:42:26.468673Z 2021-12-10T12:42:26.058637Z 2021-12-10T12:42:25.647609Z 2021-12-10T12:42:25.038638Z 2021-12-10T12:42:24.368087Z 2021-12-10T12:42:23.028598Z 2021-12-10T12:42:21.281309Z 2021-12-10T12:42:20.317707Z 2021-12-10T12:42:20.239127Z 2021-12-10T12:42:19.485050Z 2021-12-10T12:42:18.767425Z 2021-12-10T12:42:17.995284Z 2021-12-10T12:42:17.184751Z 2021-12-10T12:42:17.082020Z 2021-12-10T12:42:16.024498Z 2021-12-10T12:42:15.511206Z 2021-12-10T12:42:15.377822Z 2021-12-10T12:42:15.217416Z 2021-12-10T12:42:14.873932Z 2021-12-10T12:42:14.863460Z 2021-12-10T12:42:14.860159Z 2021-12-10T12:42:14.858790Z 2021-12-10T12:42:14.785302Z 2021-12-10T12:42:14.774926Z 2021-12-10T12:42:14.574103Z 2021-12-10T12:42:14.335936Z 2021-12-10T12:42:14.143963Z 2021-12-10T12:42:13.892021Z 2021-12-10T12:42:13.637814Z 2021-12-10T12:42:13.592305Z 2021-12-10T12:42:13.544267Z 2021-12-10T12:42:13.523260Z 2021-12-10T12:42:13.321304Z 2021-12-10T12:42:13.249277Z 2021-12-10T12:42:13.215206Z 2021-12-10T12:42:13.174061Z 2021-12-10T12:42:13.169988Z 2021-12-10T12:42:13.150746Z 2021-12-10T12:42:12.888129Z 2021-12-10T12:42:12.409886Z 2021-12-10T12:42:12.367513Z 2021-12-10T12:42:12.204518Z 2021-12-10T12:42:11.853361Z 2021-12-10T12:42:11.571439Z 2021-12-10T12:42:11.549956Z 2021-12-10T12:42:11.327304Z 2021-12-10T12:42:11.317012Z 2021-12-10T12:42:11.289428Z 2021-12-10T12:42:11.266763Z 2021-12-10T12:42:11.232525Z 2021-12-10T12:42:11.189188Z 2021-12-10T12:42:10.748512Z 2021-12-10T12:42:10.578844Z 2021-12-10T12:42:10.500112Z 2021-12-10T12:42:10.054248Z 2021-12-10T12:42:09.998896Z 2021-12-10T12:42:09.852166Z 2021-12-10T12:42:09.592047Z 2021-12-10T12:42:09.274631Z 2021-12-10T12:42:09.268764Z 2021-12-10T12:47:35.191885Z 2021-12-10T12:47:34.163886Z 2021-12-10T12:47:33.975802Z 2021-12-10T12:47:33.618992Z 2021-12-10T12:47:32.819201Z 2021-12-10T12:47:32.178671Z 2021-12-10T12:47:32.139636Z 2021-12-10T12:47:31.908149Z 2021-12-10T12:47:31.776389Z 2021-12-10T12:47:31.526531Z 2021-12-10T12:47:31.256040Z 2021-12-10T12:47:31.215576Z 2021-12-10T12:47:30.672168Z 2021-12-10T12:47:30.093998Z 2021-12-10T12:47:29.919463Z 2021-12-10T12:47:29.574503Z 2021-12-10T12:47:29.460705Z 2021-12-10T12:47:28.716636Z 2021-12-10T12:47:28.636548Z 2021-12-10T12:47:28.413174Z 2021-12-10T12:47:28.159079Z 2021-12-10T12:47:28.096151Z 2021-12-10T12:47:27.898779Z 2021-12-10T12:47:27.673312Z 2021-12-10T12:47:27.661760Z 2021-12-10T12:47:26.151030Z 2021-12-10T12:47:25.985318Z 2021-12-10T12:47:25.957209Z 2021-12-10T12:47:25.810095Z 2021-12-10T12:47:25.838667Z 2021-12-10T12:47:25.467177Z 2021-12-10T12:47:24.757998Z 2021-12-10T12:47:24.681818Z 2021-12-10T12:47:24.542860Z 2021-12-10T12:47:24.222744Z 2021-12-10T12:47:24.175085Z 2021-12-10T12:47:24.088126Z 2021-12-10T12:47:24.015114Z 2021-12-10T12:47:23.136192Z 2021-12-10T12:47:23.059109Z 2021-12-10T12:47:23.030033Z 2021-12-10T12:47:22.977504Z 2021-12-10T12:47:22.913736Z 2021-12-10T12:47:22.682197Z 2021-12-10T12:47:22.368412Z 2021-12-10T12:47:22.359493Z 2021-12-10T12:47:22.209023Z 2021-12-10T12:47:22.034712Z 2021-12-10T12:47:21.975195Z 2021-12-10T12:47:21.330555Z 2021-12-10T12:47:20.834572Z 2021-12-10T12:47:20.760869Z 2021-12-10T12:47:20.614694Z 2021-12-10T12:47:20.229758Z 2021-12-10T12:47:20.042816Z 2021-12-10T12:47:19.972584Z 2021-12-10T12:47:19.867482Z 2021-12-10T12:47:19.759769Z 2021-12-10T12:47:19.758354Z 2021-12-10T12:47:19.713519Z 2021-12-10T12:47:19.644640Z 2021-12-10T12:47:19.607216Z 2021-12-10T12:47:19.566999Z 2021-12-10T12:47:19.326518Z 2021-12-10T12:47:19.307545Z 2021-12-10T12:47:19.100339Z 2021-12-10T12:47:19.015602Z 2021-12-10T12:47:18.677183Z 2021-12-10T12:47:18.490552Z 2021-12-10T12:47:18.357304Z 2021-12-10T12:47:17.805221Z 2021-12-10T12:47:17.749311Z 2021-12-10T12:47:17.650520Z 2021-12-10T12:47:17.556983Z 2021-12-10T12:47:17.367182Z 2021-12-10T12:47:17.054606Z 2021-12-10T12:47:17.037175Z 2021-12-10T12:47:17.034047Z 2021-12-10T12:47:17.011607Z 2021-12-10T12:47:16.987761Z 2021-12-10T12:47:16.911945Z 2021-12-10T12:47:16.864707Z 2021-12-10T12:47:16.828268Z 2021-12-10T12:47:16.818150Z 2021-12-10T12:47:16.400806Z 2021-12-10T12:47:16.300502Z 2021-12-10T12:47:16.195847Z 2021-12-10T12:47:16.104754Z 2021-12-10T12:47:15.963363Z 2021-12-10T12:47:15.869367Z 2021-12-10T12:47:15.862807Z 2021-12-10T12:47:15.824161Z 2021-12-10T12:47:15.765148Z 2021-12-10T12:47:15.683302Z 2021-12-10T12:47:15.674851Z 2021-12-10T12:47:15.643317Z 2021-12-10T12:47:15.601247Z 2021-12-10T12:47:15.598680Z 2021-12-10T12:47:15.415742Z 2021-12-10T12:47:15.358974Z 2021-12-10T12:51:45.405905Z 2021-12-10T12:51:45.061225Z 2021-12-10T12:51:43.811016Z 2021-12-10T12:51:43.506720Z 2021-12-10T12:51:43.413324Z 2021-12-10T12:51:43.218955Z 2021-12-10T12:51:43.067468Z 2021-12-10T12:51:43.066358Z 2021-12-10T12:51:43.007505Z 2021-12-10T12:51:42.574391Z 2021-12-10T12:51:42.348482Z 2021-12-10T12:51:42.276912Z 2021-12-10T12:51:41.993159Z 2021-12-10T12:51:41.784119Z 2021-12-10T12:51:41.436684Z 2021-12-10T12:51:41.167979Z 2021-12-10T12:51:40.701549Z 2021-12-10T12:51:40.589754Z 2021-12-10T12:51:40.555387Z 2021-12-10T12:51:40.548461Z 2021-12-10T12:51:40.435010Z 2021-12-10T12:51:40.364182Z 2021-12-10T12:51:40.266656Z 2021-12-10T12:51:40.110784Z 2021-12-10T12:51:39.950129Z 2021-12-10T12:51:39.921615Z 2021-12-10T12:51:39.827005Z 2021-12-10T12:51:39.797762Z 2021-12-10T12:51:39.776134Z 2021-12-10T12:51:39.619005Z 2021-12-10T12:51:39.479110Z 2021-12-10T12:51:39.201599Z 2021-12-10T12:51:39.181202Z 2021-12-10T12:51:38.822555Z 2021-12-10T12:51:38.720203Z 2021-12-10T12:51:38.621152Z 2021-12-10T12:51:38.317632Z 2021-12-10T12:51:38.310358Z 2021-12-10T12:51:38.219447Z 2021-12-10T12:51:37.728456Z 2021-12-10T12:51:37.710071Z 2021-12-10T12:51:37.695104Z 2021-12-10T12:51:37.661886Z 2021-12-10T12:51:37.662920Z 2021-12-10T12:51:37.484531Z 2021-12-10T12:51:37.382744Z 2021-12-10T12:51:37.255233Z 2021-12-10T12:51:37.021052Z 2021-12-10T12:51:36.585442Z 2021-12-10T12:51:36.458144Z 2021-12-10T12:51:36.350939Z 2021-12-10T12:51:36.311371Z 2021-12-10T12:51:36.181833Z 2021-12-10T12:51:36.065684Z 2021-12-10T12:51:36.041995Z 2021-12-10T12:51:35.936507Z 2021-12-10T12:51:35.541614Z 2021-12-10T12:51:35.523982Z 2021-12-10T12:51:35.322692Z 2021-12-10T12:51:35.136278Z 2021-12-10T12:51:34.991236Z 2021-12-10T12:51:34.929285Z 2021-12-10T12:51:34.777449Z 2021-12-10T12:51:34.581659Z 2021-12-10T12:51:34.559084Z 2021-12-10T12:51:34.467865Z 2021-12-10T12:51:34.062162Z 2021-12-10T12:51:34.034341Z 2021-12-10T12:51:33.804504Z 2021-12-10T12:51:33.779212Z 2021-12-10T12:51:33.664992Z 2021-12-10T12:51:33.661975Z 2021-12-10T12:51:33.514197Z 2021-12-10T12:51:33.391569Z 2021-12-10T12:51:33.388648Z 2021-12-10T12:51:33.380712Z 2021-12-10T12:51:33.363361Z 2021-12-10T12:51:33.339970Z 2021-12-10T12:51:33.305431Z 2021-12-10T12:51:33.043980Z 2021-12-10T12:51:32.918537Z 2021-12-10T12:51:32.909062Z 2021-12-10T12:51:32.840273Z 2021-12-10T12:51:32.634909Z 2021-12-10T12:51:32.514694Z 2021-12-10T12:51:32.505930Z 2021-12-10T12:51:32.431823Z 2021-12-10T12:51:32.343486Z 2021-12-10T12:51:32.041770Z 2021-12-10T12:51:32.002998Z 2021-12-10T12:51:31.914980Z 2021-12-10T12:51:31.642564Z 2021-12-10T12:51:31.634642Z 2021-12-10T12:51:31.536230Z 2021-12-10T12:51:31.508636Z 2021-12-10T12:51:31.116086Z 2021-12-10T12:51:31.100732Z 2021-12-10T12:51:30.996637Z 2021-12-10T12:51:30.787489Z 2021-12-10T12:51:30.734998Z 2021-12-10T12:56:03.479347Z 2021-12-10T12:56:02.373707Z 2021-12-10T12:56:01.750158Z 2021-12-10T12:56:01.002730Z 2021-12-10T12:56:00.570694Z 2021-12-10T12:55:59.893231Z 2021-12-10T12:55:59.834422Z 2021-12-10T12:55:59.640696Z 2021-12-10T12:55:59.594463Z 2021-12-10T12:55:59.307409Z 2021-12-10T12:55:59.251434Z 2021-12-10T12:55:59.081514Z 2021-12-10T12:55:58.826655Z 2021-12-10T12:55:58.755841Z 2021-12-10T12:55:58.700911Z 2021-12-10T12:55:58.596260Z 2021-12-10T12:55:58.510286Z 2021-12-10T12:55:58.472568Z 2021-12-10T12:55:58.188728Z 2021-12-10T12:55:58.028293Z 2021-12-10T12:55:57.950453Z 2021-12-10T12:55:57.947563Z 2021-12-10T12:55:57.898155Z 2021-12-10T12:55:57.866147Z 2021-12-10T12:55:57.675059Z 2021-12-10T12:55:57.670742Z 2021-12-10T12:55:57.507532Z 2021-12-10T12:55:57.144638Z 2021-12-10T12:55:56.844393Z 2021-12-10T12:55:56.793773Z 2021-12-10T12:55:56.624013Z 2021-12-10T12:55:56.632583Z 2021-12-10T12:55:56.599352Z 2021-12-10T12:55:56.214288Z 2021-12-10T12:55:56.144682Z 2021-12-10T12:55:56.031837Z 2021-12-10T12:55:55.763991Z 2021-12-10T12:55:55.761908Z 2021-12-10T12:55:55.617350Z 2021-12-10T12:55:55.390155Z 2021-12-10T12:55:55.330576Z 2021-12-10T12:55:55.322420Z 2021-12-10T12:55:55.259217Z 2021-12-10T12:55:55.162071Z 2021-12-10T12:55:55.164128Z 2021-12-10T12:55:55.101701Z 2021-12-10T12:55:55.013138Z 2021-12-10T12:55:54.897348Z 2021-12-10T12:55:54.798932Z 2021-12-10T12:55:54.780267Z 2021-12-10T12:55:54.727994Z 2021-12-10T12:55:54.504932Z 2021-12-10T12:55:54.469136Z 2021-12-10T12:55:54.280171Z 2021-12-10T12:55:54.116574Z 2021-12-10T12:55:53.924122Z 2021-12-10T12:55:53.911037Z 2021-12-10T12:55:53.521545Z 2021-12-10T12:55:53.469259Z 2021-12-10T12:55:53.192072Z 2021-12-10T12:55:53.079965Z 2021-12-10T12:55:52.999063Z 2021-12-10T12:55:52.984221Z 2021-12-10T12:55:52.965841Z 2021-12-10T12:55:52.849016Z 2021-12-10T12:55:52.815971Z 2021-12-10T12:55:52.736691Z 2021-12-10T12:55:52.641178Z 2021-12-10T12:55:52.608598Z 2021-12-10T12:55:52.398371Z 2021-12-10T12:55:52.399548Z 2021-12-10T12:55:52.375981Z 2021-12-10T12:55:52.372464Z 2021-12-10T12:55:52.331012Z 2021-12-10T12:55:52.154976Z 2021-12-10T12:55:52.151842Z 2021-12-10T12:55:51.756156Z 2021-12-10T12:55:51.749464Z 2021-12-10T12:55:51.659935Z 2021-12-10T12:55:51.613437Z 2021-12-10T12:55:51.557337Z 2021-12-10T12:55:51.499022Z 2021-12-10T12:55:51.375835Z 2021-12-10T12:55:50.989514Z 2021-12-10T12:55:50.966573Z 2021-12-10T12:55:50.896773Z 2021-12-10T12:55:50.878423Z 2021-12-10T12:55:50.801035Z 2021-12-10T12:55:50.785041Z 2021-12-10T12:55:50.727795Z 2021-12-10T12:55:50.568488Z 2021-12-10T12:55:50.376672Z 2021-12-10T12:55:50.344561Z 2021-12-10T12:55:50.323434Z 2021-12-10T12:55:50.263791Z 2021-12-10T12:55:49.926761Z 2021-12-10T12:55:49.921572Z 2021-12-10T12:55:49.896608Z 2021-12-10T12:55:49.501669Z 2021-12-10T12:55:49.426541Z 2021-12-10T13:00:20.337225Z 2021-12-10T13:00:18.949144Z 2021-12-10T13:00:18.947314Z 2021-12-10T13:00:18.940722Z 2021-12-10T13:00:18.785926Z 2021-12-10T13:00:18.450699Z 2021-12-10T13:00:18.402598Z 2021-12-10T13:00:18.214660Z 2021-12-10T13:00:18.128918Z 2021-12-10T13:00:18.118075Z 2021-12-10T13:00:18.071139Z 2021-12-10T13:00:18.068171Z 2021-12-10T13:00:17.645806Z 2021-12-10T13:00:17.512122Z 2021-12-10T13:00:17.500457Z 2021-12-10T13:00:17.488070Z 2021-12-10T13:00:17.355821Z 2021-12-10T13:00:17.294698Z 2021-12-10T13:00:17.157011Z 2021-12-10T13:00:17.099764Z 2021-12-10T13:00:16.948588Z 2021-12-10T13:00:16.863873Z 2021-12-10T13:00:16.799433Z 2021-12-10T13:00:16.779291Z 2021-12-10T13:00:16.639523Z 2021-12-10T13:00:16.437003Z 2021-12-10T13:00:16.340090Z 2021-12-10T13:00:16.357598Z 2021-12-10T13:00:16.285316Z 2021-12-10T13:00:16.226881Z 2021-12-10T13:00:16.026370Z 2021-12-10T13:00:15.944533Z 2021-12-10T13:00:15.801547Z 2021-12-10T13:00:15.788680Z 2021-12-10T13:00:15.658823Z 2021-12-10T13:00:15.490454Z 2021-12-10T13:00:15.489215Z 2021-12-10T13:00:15.366775Z 2021-12-10T13:00:15.357809Z 2021-12-10T13:00:15.123293Z 2021-12-10T13:00:14.885058Z 2021-12-10T13:00:14.848610Z 2021-12-10T13:00:14.735005Z 2021-12-10T13:00:14.650387Z 2021-12-10T13:00:14.534133Z 2021-12-10T13:00:14.530551Z 2021-12-10T13:00:14.445295Z 2021-12-10T13:00:14.431941Z 2021-12-10T13:00:14.095702Z 2021-12-10T13:00:14.090215Z 2021-12-10T13:00:13.969203Z 2021-12-10T13:00:13.916594Z 2021-12-10T13:00:13.728660Z 2021-12-10T13:00:13.694875Z 2021-12-10T13:00:13.672038Z 2021-12-10T13:00:13.574650Z 2021-12-10T13:00:13.469173Z 2021-12-10T13:00:13.244110Z 2021-12-10T13:00:13.201448Z 2021-12-10T13:00:13.169843Z 2021-12-10T13:00:13.093786Z 2021-12-10T13:00:13.033044Z 2021-12-10T13:00:13.001436Z 2021-12-10T13:00:12.930578Z 2021-12-10T13:00:12.888928Z 2021-12-10T13:00:12.745494Z 2021-12-10T13:00:12.439076Z 2021-12-10T13:00:12.412573Z 2021-12-10T13:00:12.174540Z 2021-12-10T13:00:12.118442Z 2021-12-10T13:00:12.079086Z 2021-12-10T13:00:11.950526Z 2021-12-10T13:00:11.878805Z 2021-12-10T13:00:11.726429Z 2021-12-10T13:00:11.695486Z 2021-12-10T13:00:11.652371Z 2021-12-10T13:00:11.563453Z 2021-12-10T13:00:11.416609Z 2021-12-10T13:00:11.388548Z 2021-12-10T13:00:11.355090Z 2021-12-10T13:00:11.360552Z 2021-12-10T13:00:11.279041Z 2021-12-10T13:00:11.139837Z 2021-12-10T13:00:11.116007Z 2021-12-10T13:00:11.040607Z 2021-12-10T13:00:10.898121Z 2021-12-10T13:00:10.833790Z 2021-12-10T13:00:10.820688Z 2021-12-10T13:00:10.704026Z 2021-12-10T13:00:10.480437Z 2021-12-10T13:00:10.392489Z 2021-12-10T13:00:10.356326Z 2021-12-10T13:00:10.259859Z 2021-12-10T13:00:10.258648Z 2021-12-10T13:00:10.243800Z 2021-12-10T13:00:10.117605Z 2021-12-10T13:00:10.110523Z 2021-12-10T13:00:09.692449Z 2021-12-10T13:00:09.291509Z 2021-12-10T13:00:09.105519Z", + "created_diff": 17.1895354, + "ended": "2021-12-10T13:01:50.927422Z", + "error_job_ids": " []", + "event_first": "2021-12-10T12:42:38.003158Z 2021-12-10T12:43:22.888620Z 2021-12-10T12:42:41.549682Z 2021-12-10T12:43:10.593462Z 2021-12-10T12:42:42.292968Z 2021-12-10T12:43:20.064849Z 2021-12-10T12:42:42.550662Z 2021-12-10T12:43:29.752467Z 2021-12-10T12:42:29.869281Z 2021-12-10T12:43:24.652344Z 2021-12-10T12:42:36.056788Z 2021-12-10T12:43:08.416493Z 2021-12-10T12:44:45.449526Z 2021-12-10T12:43:20.318052Z 2021-12-10T12:44:26.418491Z 2021-12-10T12:43:07.307545Z 2021-12-10T12:44:17.024741Z 2021-12-10T12:43:03.347223Z 2021-12-10T12:44:54.861598Z 2021-12-10T12:43:03.034435Z 2021-12-10T12:43:58.451611Z 2021-12-10T12:43:01.890379Z 2021-12-10T12:43:27.188750Z 2021-12-10T12:42:32.335374Z 2021-12-10T12:45:04.621057Z 2021-12-10T12:42:32.851353Z 2021-12-10T12:44:36.289095Z 2021-12-10T12:42:34.065624Z 2021-12-10T12:44:08.369892Z 2021-12-10T12:42:23.370366Z 2021-12-10T12:43:40.186824Z 2021-12-10T12:42:20.923694Z 2021-12-10T12:43:50.075219Z 2021-12-10T12:42:13.603040Z 2021-12-10T12:42:31.256645Z 2021-12-10T12:42:11.044596Z 2021-12-10T12:43:27.188750Z 2021-12-10T12:43:45.277723Z 2021-12-10T12:43:30.318661Z 2021-12-10T12:43:44.979520Z 2021-12-10T12:43:30.814031Z 2021-12-10T12:43:37.514414Z 2021-12-10T12:43:28.722782Z 2021-12-10T12:43:43.929078Z 2021-12-10T12:43:23.874772Z 2021-12-10T12:43:45.908748Z 2021-12-10T12:43:28.449804Z 2021-12-10T12:43:42.986650Z 2021-12-10T12:43:26.481166Z 2021-12-10T12:43:35.252633Z 2021-12-10T12:43:44.465715Z 2021-12-10T12:43:22.592106Z 2021-12-10T12:43:45.709068Z 2021-12-10T12:43:20.774755Z 2021-12-10T12:43:33.211409Z 2021-12-10T12:43:19.710291Z 2021-12-10T12:43:45.463713Z 2021-12-10T12:43:27.449842Z 2021-12-10T12:43:39.984344Z 2021-12-10T12:43:16.555861Z 2021-12-10T12:43:40.431156Z 2021-12-10T12:43:23.656728Z 2021-12-10T12:43:37.795413Z 2021-12-10T12:43:26.232050Z 2021-12-10T12:43:30.047932Z 2021-12-10T12:43:23.368880Z 2021-12-10T12:43:35.748630Z 2021-12-10T12:43:16.012278Z 2021-12-10T12:43:35.493411Z 2021-12-10T12:43:16.834184Z 2021-12-10T12:43:28.515451Z 2021-12-10T12:43:24.635928Z 2021-12-10T12:43:43.441518Z 2021-12-10T12:43:20.529724Z 2021-12-10T12:43:37.578874Z 2021-12-10T12:43:12.353879Z 2021-12-10T12:43:33.039736Z 2021-12-10T12:43:19.181249Z 2021-12-10T12:43:40.431156Z 2021-12-10T12:43:13.065968Z 2021-12-10T12:43:38.069310Z 2021-12-10T12:43:17.063874Z 2021-12-10T12:43:36.756411Z 2021-12-10T12:43:14.136395Z 2021-12-10T12:43:39.772457Z 2021-12-10T12:43:10.794860Z 2021-12-10T12:43:24.400670Z 2021-12-10T12:43:08.438137Z 2021-12-10T12:43:38.280064Z 2021-12-10T12:43:19.684156Z 2021-12-10T12:43:36.342422Z 2021-12-10T12:43:05.510682Z 2021-12-10T12:43:26.324484Z 2021-12-10T12:43:04.823786Z 2021-12-10T12:43:38.482336Z 2021-12-10T12:43:09.455117Z 2021-12-10T12:43:30.938582Z 2021-12-10T12:43:22.360970Z 2021-12-10T12:42:52.355761Z 2021-12-10T12:43:20.368301Z 2021-12-10T12:48:50.109779Z 2021-12-10T12:48:52.846528Z 2021-12-10T12:48:53.612801Z 2021-12-10T12:48:50.464067Z 2021-12-10T12:48:52.450345Z 2021-12-10T12:48:50.461097Z 2021-12-10T12:48:53.182755Z 2021-12-10T12:48:50.464067Z 2021-12-10T12:48:55.068581Z 2021-12-10T12:48:46.364890Z 2021-12-10T12:48:53.891704Z 2021-12-10T12:48:50.982830Z 2021-12-10T12:48:51.176050Z 2021-12-10T12:48:49.470935Z 2021-12-10T12:48:52.485132Z 2021-12-10T12:48:48.872059Z 2021-12-10T12:48:54.270900Z 2021-12-10T12:48:48.120638Z 2021-12-10T12:48:51.701910Z 2021-12-10T12:48:50.461097Z 2021-12-10T12:48:48.327771Z 2021-12-10T12:48:49.635685Z 2021-12-10T12:48:46.564043Z 2021-12-10T12:48:48.872059Z 2021-12-10T12:48:51.932831Z 2021-12-10T12:48:44.982748Z 2021-12-10T12:48:48.059678Z 2021-12-10T12:48:47.611582Z 2021-12-10T12:48:46.610329Z 2021-12-10T12:48:53.885688Z 2021-12-10T12:48:49.880946Z 2021-12-10T12:48:42.132104Z 2021-12-10T12:48:52.935771Z 2021-12-10T12:48:46.811826Z 2021-12-10T12:48:44.250098Z 2021-12-10T12:48:41.230310Z 2021-12-10T12:48:39.796798Z 2021-12-10T12:48:40.625036Z 2021-12-10T12:48:40.183928Z 2021-12-10T12:48:39.946183Z 2021-12-10T12:48:40.990038Z 2021-12-10T12:48:41.474028Z 2021-12-10T12:48:35.296446Z 2021-12-10T12:48:37.922501Z 2021-12-10T12:48:42.479857Z 2021-12-10T12:48:35.448483Z 2021-12-10T12:48:35.813456Z 2021-12-10T12:48:41.027314Z 2021-12-10T12:48:32.586966Z 2021-12-10T12:48:39.183225Z 2021-12-10T12:48:39.153951Z 2021-12-10T12:48:42.385742Z 2021-12-10T12:48:31.578079Z 2021-12-10T12:48:34.334644Z 2021-12-10T12:48:32.260033Z 2021-12-10T12:48:29.502297Z 2021-12-10T12:48:25.983623Z 2021-12-10T12:48:25.939462Z 2021-12-10T12:48:25.719835Z 2021-12-10T12:48:35.649755Z 2021-12-10T12:48:27.227763Z 2021-12-10T12:48:27.220336Z 2021-12-10T12:48:22.444777Z 2021-12-10T12:48:28.971674Z 2021-12-10T12:48:29.289650Z 2021-12-10T12:48:19.503254Z 2021-12-10T12:48:27.231116Z 2021-12-10T12:48:24.915282Z 2021-12-10T12:48:27.689473Z 2021-12-10T12:48:19.012693Z 2021-12-10T12:48:20.412884Z 2021-12-10T12:48:25.600383Z 2021-12-10T12:48:22.444777Z 2021-12-10T12:48:22.100778Z 2021-12-10T12:48:21.577184Z 2021-12-10T12:47:55.231941Z 2021-12-10T12:47:58.281029Z 2021-12-10T12:48:15.042649Z 2021-12-10T12:48:15.702394Z 2021-12-10T12:48:14.252026Z 2021-12-10T12:47:45.071971Z 2021-12-10T12:47:50.757689Z 2021-12-10T12:47:47.089932Z 2021-12-10T12:47:46.503635Z 2021-12-10T12:47:43.564102Z 2021-12-10T12:47:51.399444Z 2021-12-10T12:47:44.034877Z 2021-12-10T12:47:42.832710Z 2021-12-10T12:47:39.593013Z 2021-12-10T12:47:39.099429Z 2021-12-10T12:47:40.383269Z 2021-12-10T12:47:39.865930Z 2021-12-10T12:47:36.446795Z 2021-12-10T12:47:30.252561Z 2021-12-10T12:47:41.403117Z 2021-12-10T12:47:37.793350Z 2021-12-10T12:47:25.373293Z 2021-12-10T12:47:25.545170Z 2021-12-10T12:47:24.427187Z 2021-12-10T12:47:28.872596Z 2021-12-10T12:53:08.572549Z 2021-12-10T12:53:09.247500Z 2021-12-10T12:53:04.331987Z 2021-12-10T12:53:06.360364Z 2021-12-10T12:53:04.781271Z 2021-12-10T12:53:06.589372Z 2021-12-10T12:53:08.785187Z 2021-12-10T12:53:10.836435Z 2021-12-10T12:52:58.328729Z 2021-12-10T12:53:07.851621Z 2021-12-10T12:53:06.183820Z 2021-12-10T12:53:04.944788Z 2021-12-10T12:53:04.781271Z 2021-12-10T12:53:02.902240Z 2021-12-10T12:53:04.882201Z 2021-12-10T12:53:05.651531Z 2021-12-10T12:53:04.331987Z 2021-12-10T12:53:03.365074Z 2021-12-10T12:53:02.050940Z 2021-12-10T12:53:03.101657Z 2021-12-10T12:53:01.634899Z 2021-12-10T12:53:08.135329Z 2021-12-10T12:53:02.905744Z 2021-12-10T12:53:04.895244Z 2021-12-10T12:53:05.755927Z 2021-12-10T12:53:08.638659Z 2021-12-10T12:53:02.042614Z 2021-12-10T12:52:56.001718Z 2021-12-10T12:53:00.030001Z 2021-12-10T12:53:00.385423Z 2021-12-10T12:52:55.084886Z 2021-12-10T12:53:05.908503Z 2021-12-10T12:53:01.283799Z 2021-12-10T12:53:03.923563Z 2021-12-10T12:53:05.183439Z 2021-12-10T12:53:01.140431Z 2021-12-10T12:52:58.077783Z 2021-12-10T12:53:03.366319Z 2021-12-10T12:52:55.942685Z 2021-12-10T12:53:01.653357Z 2021-12-10T12:52:55.041890Z 2021-12-10T12:52:56.473443Z 2021-12-10T12:52:55.288160Z 2021-12-10T12:52:58.518899Z 2021-12-10T12:52:59.072207Z 2021-12-10T12:53:02.913731Z 2021-12-10T12:52:55.146895Z 2021-12-10T12:52:59.611189Z 2021-12-10T12:52:51.507749Z 2021-12-10T12:52:54.186209Z 2021-12-10T12:52:48.542612Z 2021-12-10T12:52:55.856876Z 2021-12-10T12:52:56.993714Z 2021-12-10T12:52:57.069348Z 2021-12-10T12:52:51.952874Z 2021-12-10T12:52:48.523211Z 2021-12-10T12:52:52.605098Z 2021-12-10T12:52:57.069348Z 2021-12-10T12:52:48.352167Z 2021-12-10T12:52:50.782767Z 2021-12-10T12:52:30.510903Z 2021-12-10T12:52:34.455596Z 2021-12-10T12:52:33.685790Z 2021-12-10T12:52:28.529326Z 2021-12-10T12:52:27.799923Z 2021-12-10T12:52:30.938565Z 2021-12-10T12:52:28.248559Z 2021-12-10T12:52:31.441576Z 2021-12-10T12:52:21.512769Z 2021-12-10T12:52:23.785888Z 2021-12-10T12:52:30.012180Z 2021-12-10T12:52:34.701849Z 2021-12-10T12:52:29.009073Z 2021-12-10T12:52:20.518506Z 2021-12-10T12:52:19.206243Z 2021-12-10T12:52:15.937024Z 2021-12-10T12:52:12.803932Z 2021-12-10T12:52:21.825637Z 2021-12-10T12:52:13.340786Z 2021-12-10T12:52:12.660444Z 2021-12-10T12:52:12.295184Z 2021-12-10T12:52:07.010104Z 2021-12-10T12:52:12.299915Z 2021-12-10T12:52:11.025273Z 2021-12-10T12:52:11.093522Z 2021-12-10T12:52:08.110877Z 2021-12-10T12:52:01.823791Z 2021-12-10T12:52:01.240621Z 2021-12-10T12:52:05.973275Z 2021-12-10T12:52:08.363038Z 2021-12-10T12:51:59.911545Z 2021-12-10T12:51:48.872643Z 2021-12-10T12:51:47.164869Z 2021-12-10T12:51:48.642255Z 2021-12-10T12:51:45.700449Z 2021-12-10T12:51:42.831454Z 2021-12-10T12:51:41.819993Z 2021-12-10T12:51:44.416537Z 2021-12-10T12:51:38.380805Z 2021-12-10T12:51:41.588860Z 2021-12-10T12:57:30.189471Z 2021-12-10T12:57:27.208113Z 2021-12-10T12:57:30.019233Z 2021-12-10T12:57:27.458777Z 2021-12-10T12:57:28.943322Z 2021-12-10T12:57:23.346315Z 2021-12-10T12:57:22.540217Z 2021-12-10T12:57:26.996193Z 2021-12-10T12:57:19.712790Z 2021-12-10T12:57:25.349445Z 2021-12-10T12:57:23.047848Z 2021-12-10T12:57:22.090578Z 2021-12-10T12:57:20.216758Z 2021-12-10T12:57:23.893149Z 2021-12-10T12:57:23.002945Z 2021-12-10T12:57:21.584818Z 2021-12-10T12:57:17.435748Z 2021-12-10T12:57:23.322448Z 2021-12-10T12:57:24.515274Z 2021-12-10T12:57:19.573746Z 2021-12-10T12:57:23.177542Z 2021-12-10T12:57:28.014345Z 2021-12-10T12:57:17.435748Z 2021-12-10T12:57:26.719335Z 2021-12-10T12:57:22.411949Z 2021-12-10T12:57:23.577394Z 2021-12-10T12:57:24.519592Z 2021-12-10T12:57:25.089332Z 2021-12-10T12:57:25.221216Z 2021-12-10T12:57:23.033394Z 2021-12-10T12:57:14.107577Z 2021-12-10T12:57:18.191791Z 2021-12-10T12:57:17.201763Z 2021-12-10T12:57:13.728497Z 2021-12-10T12:57:17.964834Z 2021-12-10T12:57:24.623241Z 2021-12-10T12:57:16.231986Z 2021-12-10T12:57:20.383081Z 2021-12-10T12:57:17.031789Z 2021-12-10T12:57:18.504656Z 2021-12-10T12:57:17.201763Z 2021-12-10T12:57:11.397621Z 2021-12-10T12:57:17.202602Z 2021-12-10T12:57:14.435754Z 2021-12-10T12:57:23.346315Z 2021-12-10T12:57:18.080960Z 2021-12-10T12:57:18.981622Z 2021-12-10T12:57:15.665981Z 2021-12-10T12:57:10.051986Z 2021-12-10T12:57:22.778584Z 2021-12-10T12:57:14.107806Z 2021-12-10T12:57:08.586451Z 2021-12-10T12:57:12.992229Z 2021-12-10T12:57:11.914680Z 2021-12-10T12:57:11.645431Z 2021-12-10T12:57:14.778445Z 2021-12-10T12:57:09.581868Z 2021-12-10T12:57:12.704979Z 2021-12-10T12:57:11.228000Z 2021-12-10T12:57:10.057826Z 2021-12-10T12:57:07.877907Z 2021-12-10T12:57:09.801667Z 2021-12-10T12:57:01.345855Z 2021-12-10T12:57:01.250725Z 2021-12-10T12:56:58.126208Z 2021-12-10T12:56:56.041446Z 2021-12-10T12:56:50.286906Z 2021-12-10T12:56:59.531346Z 2021-12-10T12:57:02.370865Z 2021-12-10T12:56:50.061866Z 2021-12-10T12:57:01.028451Z 2021-12-10T12:56:52.378576Z 2021-12-10T12:57:01.536911Z 2021-12-10T12:56:49.313171Z 2021-12-10T12:56:57.870278Z 2021-12-10T12:56:57.684802Z 2021-12-10T12:57:01.345855Z 2021-12-10T12:56:42.915538Z 2021-12-10T12:56:50.370932Z 2021-12-10T12:56:49.589613Z 2021-12-10T12:56:47.547580Z 2021-12-10T12:56:37.759237Z 2021-12-10T12:56:41.779041Z 2021-12-10T12:56:13.490490Z 2021-12-10T12:56:19.592943Z 2021-12-10T12:56:14.292757Z 2021-12-10T12:56:14.192906Z 2021-12-10T12:56:13.586331Z 2021-12-10T12:56:15.640952Z 2021-12-10T12:56:13.130305Z 2021-12-10T12:56:11.420768Z 2021-12-10T12:56:11.700688Z 2021-12-10T12:56:13.339931Z 2021-12-10T12:56:11.548443Z 2021-12-10T12:56:09.235992Z 2021-12-10T12:56:05.790186Z 2021-12-10T12:56:00.465256Z 2021-12-10T12:56:02.519239Z 2021-12-10T12:55:57.807160Z 2021-12-10T12:55:59.739755Z 2021-12-10T13:01:50.200738Z 2021-12-10T13:01:46.728462Z 2021-12-10T13:01:46.706719Z 2021-12-10T13:01:45.128413Z 2021-12-10T13:01:47.932694Z 2021-12-10T13:01:44.619693Z 2021-12-10T13:01:47.667068Z 2021-12-10T13:01:41.351721Z 2021-12-10T13:01:45.767162Z 2021-12-10T13:01:44.628165Z 2021-12-10T13:01:47.667068Z 2021-12-10T13:01:44.365447Z 2021-12-10T13:01:42.098759Z 2021-12-10T13:01:36.506148Z 2021-12-10T13:01:46.148111Z 2021-12-10T13:01:47.486465Z 2021-12-10T13:01:44.609320Z 2021-12-10T13:01:41.675534Z 2021-12-10T13:01:42.355216Z 2021-12-10T13:01:43.685175Z 2021-12-10T13:01:46.236102Z 2021-12-10T13:01:37.770184Z 2021-12-10T13:01:41.185890Z 2021-12-10T13:01:37.123826Z 2021-12-10T13:01:42.098759Z 2021-12-10T13:01:37.533851Z 2021-12-10T13:01:42.864308Z 2021-12-10T13:01:42.962908Z 2021-12-10T13:01:38.395769Z 2021-12-10T13:01:40.349694Z 2021-12-10T13:01:45.269752Z 2021-12-10T13:01:32.989516Z 2021-12-10T13:01:32.644468Z 2021-12-10T13:01:40.225358Z 2021-12-10T13:01:39.862159Z 2021-12-10T13:01:43.123527Z 2021-12-10T13:01:40.702461Z 2021-12-10T13:01:32.996588Z 2021-12-10T13:01:35.204608Z 2021-12-10T13:01:39.390546Z 2021-12-10T13:01:36.945851Z 2021-12-10T13:01:34.729435Z 2021-12-10T13:01:43.473763Z 2021-12-10T13:01:38.530229Z 2021-12-10T13:01:37.659860Z 2021-12-10T13:01:36.868316Z 2021-12-10T13:01:34.908996Z 2021-12-10T13:01:33.741746Z 2021-12-10T13:01:38.147794Z 2021-12-10T13:01:28.078606Z 2021-12-10T13:01:37.446715Z 2021-12-10T13:01:26.219804Z 2021-12-10T13:01:31.122804Z 2021-12-10T13:01:31.280318Z 2021-12-10T13:01:28.620743Z 2021-12-10T13:01:29.871579Z 2021-12-10T13:01:33.393886Z 2021-12-10T13:01:23.081781Z 2021-12-10T13:01:27.332913Z 2021-12-10T13:01:24.938402Z 2021-12-10T13:01:30.316843Z 2021-12-10T13:01:24.267477Z 2021-12-10T13:01:32.639983Z 2021-12-10T13:01:23.257320Z 2021-12-10T13:01:25.526885Z 2021-12-10T13:01:23.907447Z 2021-12-10T13:01:19.268744Z 2021-12-10T13:01:20.574737Z 2021-12-10T13:01:19.023450Z 2021-12-10T13:01:25.659335Z 2021-12-10T13:01:27.614166Z 2021-12-10T13:01:24.533746Z 2021-12-10T13:01:20.002765Z 2021-12-10T13:01:21.738472Z 2021-12-10T13:01:02.777807Z 2021-12-10T13:00:57.447590Z 2021-12-10T13:00:53.701454Z 2021-12-10T13:00:57.500724Z 2021-12-10T13:00:58.232568Z 2021-12-10T13:00:43.196761Z 2021-12-10T13:00:55.947003Z 2021-12-10T13:00:44.196986Z 2021-12-10T13:00:58.155057Z 2021-12-10T13:00:49.313255Z 2021-12-10T13:00:49.857887Z 2021-12-10T13:00:40.237558Z 2021-12-10T13:00:50.176651Z 2021-12-10T13:00:41.894309Z 2021-12-10T13:00:34.508781Z 2021-12-10T13:00:37.262473Z 2021-12-10T13:00:38.172176Z 2021-12-10T13:00:38.432455Z 2021-12-10T13:00:37.198847Z 2021-12-10T13:00:35.594489Z 2021-12-10T13:00:44.240876Z 2021-12-10T13:00:33.496515Z 2021-12-10T13:00:34.825205Z 2021-12-10T13:00:22.003930Z 2021-12-10T13:00:16.176968Z 2021-12-10T13:00:18.523991Z", + "event_last": "2021-12-10T12:42:41.538447Z 2021-12-10T12:43:26.744516Z 2021-12-10T12:42:44.507209Z 2021-12-10T12:43:14.801235Z 2021-12-10T12:42:45.794569Z 2021-12-10T12:43:23.892149Z 2021-12-10T12:42:47.769144Z 2021-12-10T12:43:34.995250Z 2021-12-10T12:42:32.617628Z 2021-12-10T12:43:29.576159Z 2021-12-10T12:42:40.657795Z 2021-12-10T12:43:12.657193Z 2021-12-10T12:44:45.979340Z 2021-12-10T12:43:24.414465Z 2021-12-10T12:44:26.952814Z 2021-12-10T12:43:10.955147Z 2021-12-10T12:44:17.863251Z 2021-12-10T12:43:07.733208Z 2021-12-10T12:44:55.016058Z 2021-12-10T12:43:07.524067Z 2021-12-10T12:43:59.057119Z 2021-12-10T12:43:04.859881Z 2021-12-10T12:43:29.568175Z 2021-12-10T12:42:35.794680Z 2021-12-10T12:45:07.176737Z 2021-12-10T12:42:36.316315Z 2021-12-10T12:44:36.693170Z 2021-12-10T12:42:37.074858Z 2021-12-10T12:44:08.789152Z 2021-12-10T12:42:25.903286Z 2021-12-10T12:43:41.915749Z 2021-12-10T12:42:22.748172Z 2021-12-10T12:43:50.770078Z 2021-12-10T12:42:14.950897Z 2021-12-10T12:42:33.944389Z 2021-12-10T12:42:12.190886Z 2021-12-10T12:43:29.212309Z 2021-12-10T12:43:47.643412Z 2021-12-10T12:43:31.007167Z 2021-12-10T12:43:47.567760Z 2021-12-10T12:43:32.029991Z 2021-12-10T12:43:40.727330Z 2021-12-10T12:43:30.529065Z 2021-12-10T12:43:46.596340Z 2021-12-10T12:43:26.548887Z 2021-12-10T12:43:48.167226Z 2021-12-10T12:43:30.629033Z 2021-12-10T12:43:45.144316Z 2021-12-10T12:43:28.929029Z 2021-12-10T12:43:39.366343Z 2021-12-10T12:43:47.680708Z 2021-12-10T12:43:25.746035Z 2021-12-10T12:43:50.339042Z 2021-12-10T12:43:23.414531Z 2021-12-10T12:43:36.508020Z 2021-12-10T12:43:23.311823Z 2021-12-10T12:43:47.567760Z 2021-12-10T12:43:29.651561Z 2021-12-10T12:43:44.337269Z 2021-12-10T12:43:19.741582Z 2021-12-10T12:43:43.786488Z 2021-12-10T12:43:26.648936Z 2021-12-10T12:43:42.100217Z 2021-12-10T12:43:28.859504Z 2021-12-10T12:43:35.301686Z 2021-12-10T12:43:26.648936Z 2021-12-10T12:43:39.641584Z 2021-12-10T12:43:19.480781Z 2021-12-10T12:43:39.027237Z 2021-12-10T12:43:20.284621Z 2021-12-10T12:43:33.034653Z 2021-12-10T12:43:27.333279Z 2021-12-10T12:43:45.763185Z 2021-12-10T12:43:23.412626Z 2021-12-10T12:43:41.131527Z 2021-12-10T12:43:15.711566Z 2021-12-10T12:43:37.893690Z 2021-12-10T12:43:22.985627Z 2021-12-10T12:43:43.603190Z 2021-12-10T12:43:15.797583Z 2021-12-10T12:43:42.901167Z 2021-12-10T12:43:20.146605Z 2021-12-10T12:43:40.397129Z 2021-12-10T12:43:17.681640Z 2021-12-10T12:43:43.259206Z 2021-12-10T12:43:14.099641Z 2021-12-10T12:43:27.786471Z 2021-12-10T12:43:11.599590Z 2021-12-10T12:43:41.847195Z 2021-12-10T12:43:22.666183Z 2021-12-10T12:43:39.514135Z 2021-12-10T12:43:08.970689Z 2021-12-10T12:43:29.958292Z 2021-12-10T12:43:09.250630Z 2021-12-10T12:43:41.763602Z 2021-12-10T12:43:13.315588Z 2021-12-10T12:43:34.995250Z 2021-12-10T12:43:27.396218Z 2021-12-10T12:42:55.642693Z 2021-12-10T12:43:24.919208Z 2021-12-10T12:48:51.799783Z 2021-12-10T12:48:55.883468Z 2021-12-10T12:48:55.613725Z 2021-12-10T12:48:52.538604Z 2021-12-10T12:48:54.310654Z 2021-12-10T12:48:52.273882Z 2021-12-10T12:48:55.677430Z 2021-12-10T12:48:52.922842Z 2021-12-10T12:48:58.571127Z 2021-12-10T12:48:49.111033Z 2021-12-10T12:48:56.029503Z 2021-12-10T12:48:52.944252Z 2021-12-10T12:48:52.862868Z 2021-12-10T12:48:52.185581Z 2021-12-10T12:48:55.167823Z 2021-12-10T12:48:51.149069Z 2021-12-10T12:48:55.935807Z 2021-12-10T12:48:50.464191Z 2021-12-10T12:48:53.472951Z 2021-12-10T12:48:52.833861Z 2021-12-10T12:48:50.825229Z 2021-12-10T12:48:52.406458Z 2021-12-10T12:48:48.334623Z 2021-12-10T12:48:50.966758Z 2021-12-10T12:48:53.885688Z 2021-12-10T12:48:47.409214Z 2021-12-10T12:48:49.892626Z 2021-12-10T12:48:49.806434Z 2021-12-10T12:48:49.332206Z 2021-12-10T12:48:55.235009Z 2021-12-10T12:48:52.039604Z 2021-12-10T12:48:45.965291Z 2021-12-10T12:48:54.374167Z 2021-12-10T12:48:49.490673Z 2021-12-10T12:48:47.066624Z 2021-12-10T12:48:43.861994Z 2021-12-10T12:48:42.434597Z 2021-12-10T12:48:42.890327Z 2021-12-10T12:48:43.495703Z 2021-12-10T12:48:43.075167Z 2021-12-10T12:48:44.050566Z 2021-12-10T12:48:45.177158Z 2021-12-10T12:48:39.238902Z 2021-12-10T12:48:40.987178Z 2021-12-10T12:48:44.770720Z 2021-12-10T12:48:38.273745Z 2021-12-10T12:48:39.055616Z 2021-12-10T12:48:44.116452Z 2021-12-10T12:48:36.175606Z 2021-12-10T12:48:42.336662Z 2021-12-10T12:48:42.424676Z 2021-12-10T12:48:44.885403Z 2021-12-10T12:48:36.171001Z 2021-12-10T12:48:37.138876Z 2021-12-10T12:48:36.666695Z 2021-12-10T12:48:33.876430Z 2021-12-10T12:48:31.157143Z 2021-12-10T12:48:30.188187Z 2021-12-10T12:48:29.036866Z 2021-12-10T12:48:38.280203Z 2021-12-10T12:48:31.154951Z 2021-12-10T12:48:30.537498Z 2021-12-10T12:48:26.825665Z 2021-12-10T12:48:33.047261Z 2021-12-10T12:48:33.499721Z 2021-12-10T12:48:23.494163Z 2021-12-10T12:48:32.245126Z 2021-12-10T12:48:27.893391Z 2021-12-10T12:48:31.938832Z 2021-12-10T12:48:22.750178Z 2021-12-10T12:48:24.401121Z 2021-12-10T12:48:31.018184Z 2021-12-10T12:48:26.441610Z 2021-12-10T12:48:26.202362Z 2021-12-10T12:48:24.925735Z 2021-12-10T12:47:58.273328Z 2021-12-10T12:48:01.764146Z 2021-12-10T12:48:19.250143Z 2021-12-10T12:48:19.553881Z 2021-12-10T12:48:17.845570Z 2021-12-10T12:47:47.272953Z 2021-12-10T12:47:53.293362Z 2021-12-10T12:47:49.572803Z 2021-12-10T12:47:48.549680Z 2021-12-10T12:47:45.184694Z 2021-12-10T12:47:54.193727Z 2021-12-10T12:47:46.422455Z 2021-12-10T12:47:44.721521Z 2021-12-10T12:47:41.563696Z 2021-12-10T12:47:41.219603Z 2021-12-10T12:47:41.730598Z 2021-12-10T12:47:41.958241Z 2021-12-10T12:47:37.454551Z 2021-12-10T12:47:32.069545Z 2021-12-10T12:47:43.453366Z 2021-12-10T12:47:39.130937Z 2021-12-10T12:47:26.786267Z 2021-12-10T12:47:27.101191Z 2021-12-10T12:47:26.423653Z 2021-12-10T12:47:30.611453Z 2021-12-10T12:53:09.565463Z 2021-12-10T12:53:10.838766Z 2021-12-10T12:53:07.061596Z 2021-12-10T12:53:09.408805Z 2021-12-10T12:53:07.714561Z 2021-12-10T12:53:09.888370Z 2021-12-10T12:53:09.565463Z 2021-12-10T12:53:11.484222Z 2021-12-10T12:53:02.305362Z 2021-12-10T12:53:09.853740Z 2021-12-10T12:53:08.572549Z 2021-12-10T12:53:07.794177Z 2021-12-10T12:53:07.474344Z 2021-12-10T12:53:05.995624Z 2021-12-10T12:53:07.518886Z 2021-12-10T12:53:09.241869Z 2021-12-10T12:53:06.964538Z 2021-12-10T12:53:07.502764Z 2021-12-10T12:53:05.184560Z 2021-12-10T12:53:07.294218Z 2021-12-10T12:53:04.520564Z 2021-12-10T12:53:10.315734Z 2021-12-10T12:53:05.491284Z 2021-12-10T12:53:08.720234Z 2021-12-10T12:53:08.173661Z 2021-12-10T12:53:10.836435Z 2021-12-10T12:53:04.336361Z 2021-12-10T12:52:59.208164Z 2021-12-10T12:53:04.270954Z 2021-12-10T12:53:04.137706Z 2021-12-10T12:52:58.286767Z 2021-12-10T12:53:08.620087Z 2021-12-10T12:53:04.196159Z 2021-12-10T12:53:07.419222Z 2021-12-10T12:53:07.781665Z 2021-12-10T12:53:04.795467Z 2021-12-10T12:53:01.816358Z 2021-12-10T12:53:06.244157Z 2021-12-10T12:52:58.814257Z 2021-12-10T12:53:05.624069Z 2021-12-10T12:52:57.823772Z 2021-12-10T12:52:59.861342Z 2021-12-10T12:52:57.780169Z 2021-12-10T12:53:01.786521Z 2021-12-10T12:53:02.451182Z 2021-12-10T12:53:08.048475Z 2021-12-10T12:52:58.814257Z 2021-12-10T12:53:02.898961Z 2021-12-10T12:52:55.400907Z 2021-12-10T12:52:56.717151Z 2021-12-10T12:52:52.571428Z 2021-12-10T12:52:59.548159Z 2021-12-10T12:53:00.514663Z 2021-12-10T12:53:00.459293Z 2021-12-10T12:52:54.996641Z 2021-12-10T12:52:51.615135Z 2021-12-10T12:52:54.827032Z 2021-12-10T12:53:00.184591Z 2021-12-10T12:52:52.511583Z 2021-12-10T12:52:54.577014Z 2021-12-10T12:52:34.643339Z 2021-12-10T12:52:38.467171Z 2021-12-10T12:52:37.708305Z 2021-12-10T12:52:32.861569Z 2021-12-10T12:52:32.322740Z 2021-12-10T12:52:34.645971Z 2021-12-10T12:52:31.694514Z 2021-12-10T12:52:35.772057Z 2021-12-10T12:52:25.612955Z 2021-12-10T12:52:29.275024Z 2021-12-10T12:52:33.810879Z 2021-12-10T12:52:38.100551Z 2021-12-10T12:52:34.081235Z 2021-12-10T12:52:25.758394Z 2021-12-10T12:52:24.201477Z 2021-12-10T12:52:20.288334Z 2021-12-10T12:52:16.878872Z 2021-12-10T12:52:27.140130Z 2021-12-10T12:52:17.624601Z 2021-12-10T12:52:17.323605Z 2021-12-10T12:52:16.037642Z 2021-12-10T12:52:11.423530Z 2021-12-10T12:52:16.748059Z 2021-12-10T12:52:15.144748Z 2021-12-10T12:52:15.338070Z 2021-12-10T12:52:10.758711Z 2021-12-10T12:52:04.173165Z 2021-12-10T12:52:03.004175Z 2021-12-10T12:52:08.795954Z 2021-12-10T12:52:10.968522Z 2021-12-10T12:52:02.247621Z 2021-12-10T12:51:50.451197Z 2021-12-10T12:51:48.011186Z 2021-12-10T12:51:50.269086Z 2021-12-10T12:51:46.696888Z 2021-12-10T12:51:44.433731Z 2021-12-10T12:51:42.960456Z 2021-12-10T12:51:45.115768Z 2021-12-10T12:51:39.712506Z 2021-12-10T12:51:42.931637Z 2021-12-10T12:57:32.891198Z 2021-12-10T12:57:27.805662Z 2021-12-10T12:57:33.191206Z 2021-12-10T12:57:29.641502Z 2021-12-10T12:57:29.998906Z 2021-12-10T12:57:26.031195Z 2021-12-10T12:57:25.033838Z 2021-12-10T12:57:29.035391Z 2021-12-10T12:57:22.842949Z 2021-12-10T12:57:28.407189Z 2021-12-10T12:57:25.693086Z 2021-12-10T12:57:25.209335Z 2021-12-10T12:57:23.732308Z 2021-12-10T12:57:26.331384Z 2021-12-10T12:57:25.501026Z 2021-12-10T12:57:24.972147Z 2021-12-10T12:57:22.072604Z 2021-12-10T12:57:28.082063Z 2021-12-10T12:57:26.465445Z 2021-12-10T12:57:22.682512Z 2021-12-10T12:57:25.256208Z 2021-12-10T12:57:29.172418Z 2021-12-10T12:57:20.610860Z 2021-12-10T12:57:28.569161Z 2021-12-10T12:57:24.596707Z 2021-12-10T12:57:26.331384Z 2021-12-10T12:57:26.197607Z 2021-12-10T12:57:28.265261Z 2021-12-10T12:57:27.213939Z 2021-12-10T12:57:26.501169Z 2021-12-10T12:57:17.650248Z 2021-12-10T12:57:21.400664Z 2021-12-10T12:57:21.033804Z 2021-12-10T12:57:18.168028Z 2021-12-10T12:57:21.861656Z 2021-12-10T12:57:27.448306Z 2021-12-10T12:57:19.480665Z 2021-12-10T12:57:23.526213Z 2021-12-10T12:57:21.536740Z 2021-12-10T12:57:22.303473Z 2021-12-10T12:57:20.606151Z 2021-12-10T12:57:15.228205Z 2021-12-10T12:57:20.606151Z 2021-12-10T12:57:17.790720Z 2021-12-10T12:57:26.331384Z 2021-12-10T12:57:20.811218Z 2021-12-10T12:57:22.965625Z 2021-12-10T12:57:19.168285Z 2021-12-10T12:57:13.575118Z 2021-12-10T12:57:25.877850Z 2021-12-10T12:57:17.395784Z 2021-12-10T12:57:11.947037Z 2021-12-10T12:57:16.861785Z 2021-12-10T12:57:15.582179Z 2021-12-10T12:57:16.026561Z 2021-12-10T12:57:17.768180Z 2021-12-10T12:57:12.688578Z 2021-12-10T12:57:16.335161Z 2021-12-10T12:57:14.611593Z 2021-12-10T12:57:13.725206Z 2021-12-10T12:57:11.187954Z 2021-12-10T12:57:13.645402Z 2021-12-10T12:57:05.137622Z 2021-12-10T12:57:05.934353Z 2021-12-10T12:57:01.539602Z 2021-12-10T12:57:00.633706Z 2021-12-10T12:56:54.744660Z 2021-12-10T12:57:04.223215Z 2021-12-10T12:57:06.466591Z 2021-12-10T12:56:53.995025Z 2021-12-10T12:57:06.367042Z 2021-12-10T12:56:56.429154Z 2021-12-10T12:57:06.935705Z 2021-12-10T12:56:53.720877Z 2021-12-10T12:57:01.435612Z 2021-12-10T12:57:00.516886Z 2021-12-10T12:57:06.124675Z 2021-12-10T12:56:46.926193Z 2021-12-10T12:56:54.459517Z 2021-12-10T12:56:53.991189Z 2021-12-10T12:56:51.739685Z 2021-12-10T12:56:41.861156Z 2021-12-10T12:56:45.794576Z 2021-12-10T12:56:15.722059Z 2021-12-10T12:56:22.380161Z 2021-12-10T12:56:19.359566Z 2021-12-10T12:56:16.603604Z 2021-12-10T12:56:16.171526Z 2021-12-10T12:56:17.899736Z 2021-12-10T12:56:15.309799Z 2021-12-10T12:56:12.976596Z 2021-12-10T12:56:14.501994Z 2021-12-10T12:56:15.336763Z 2021-12-10T12:56:13.358243Z 2021-12-10T12:56:10.455641Z 2021-12-10T12:56:06.755399Z 2021-12-10T12:56:03.756580Z 2021-12-10T12:56:03.707403Z 2021-12-10T12:55:59.214835Z 2021-12-10T12:56:00.810163Z 2021-12-10T13:01:50.927422Z 2021-12-10T13:01:47.489259Z 2021-12-10T13:01:48.649894Z 2021-12-10T13:01:47.082260Z 2021-12-10T13:01:50.170310Z 2021-12-10T13:01:46.660738Z 2021-12-10T13:01:49.954588Z 2021-12-10T13:01:43.530629Z 2021-12-10T13:01:48.099579Z 2021-12-10T13:01:46.683116Z 2021-12-10T13:01:49.689601Z 2021-12-10T13:01:46.653163Z 2021-12-10T13:01:45.789540Z 2021-12-10T13:01:39.642167Z 2021-12-10T13:01:48.711647Z 2021-12-10T13:01:48.192585Z 2021-12-10T13:01:46.941611Z 2021-12-10T13:01:44.598447Z 2021-12-10T13:01:45.705711Z 2021-12-10T13:01:45.715219Z 2021-12-10T13:01:48.816653Z 2021-12-10T13:01:40.935969Z 2021-12-10T13:01:44.758550Z 2021-12-10T13:01:40.080958Z 2021-12-10T13:01:45.641195Z 2021-12-10T13:01:40.150167Z 2021-12-10T13:01:44.791183Z 2021-12-10T13:01:46.860076Z 2021-12-10T13:01:41.015891Z 2021-12-10T13:01:43.038622Z 2021-12-10T13:01:48.388190Z 2021-12-10T13:01:37.022159Z 2021-12-10T13:01:35.406923Z 2021-12-10T13:01:42.319516Z 2021-12-10T13:01:42.441630Z 2021-12-10T13:01:45.004165Z 2021-12-10T13:01:43.553587Z 2021-12-10T13:01:36.635590Z 2021-12-10T13:01:39.284576Z 2021-12-10T13:01:41.799278Z 2021-12-10T13:01:39.433620Z 2021-12-10T13:01:39.385772Z 2021-12-10T13:01:46.972906Z 2021-12-10T13:01:41.611559Z 2021-12-10T13:01:41.263635Z 2021-12-10T13:01:40.006290Z 2021-12-10T13:01:39.241051Z 2021-12-10T13:01:36.083197Z 2021-12-10T13:01:42.741080Z 2021-12-10T13:01:31.357555Z 2021-12-10T13:01:41.389303Z 2021-12-10T13:01:31.236084Z 2021-12-10T13:01:34.477049Z 2021-12-10T13:01:34.809414Z 2021-12-10T13:01:32.483620Z 2021-12-10T13:01:32.744918Z 2021-12-10T13:01:37.480656Z 2021-12-10T13:01:27.192171Z 2021-12-10T13:01:31.703000Z 2021-12-10T13:01:29.151071Z 2021-12-10T13:01:32.949098Z 2021-12-10T13:01:29.198105Z 2021-12-10T13:01:36.930149Z 2021-12-10T13:01:27.033485Z 2021-12-10T13:01:28.737135Z 2021-12-10T13:01:28.100352Z 2021-12-10T13:01:21.974299Z 2021-12-10T13:01:23.344587Z 2021-12-10T13:01:22.607584Z 2021-12-10T13:01:29.974682Z 2021-12-10T13:01:31.479839Z 2021-12-10T13:01:28.213206Z 2021-12-10T13:01:23.978203Z 2021-12-10T13:01:24.822232Z 2021-12-10T13:01:07.653180Z 2021-12-10T13:01:01.854476Z 2021-12-10T13:00:58.764697Z 2021-12-10T13:01:01.573225Z 2021-12-10T13:01:02.245656Z 2021-12-10T13:00:47.437656Z 2021-12-10T13:01:01.339904Z 2021-12-10T13:00:49.414070Z 2021-12-10T13:01:04.044810Z 2021-12-10T13:00:54.318316Z 2021-12-10T13:00:56.821038Z 2021-12-10T13:00:44.025629Z 2021-12-10T13:00:55.807491Z 2021-12-10T13:00:46.338192Z 2021-12-10T13:00:37.214460Z 2021-12-10T13:00:39.853218Z 2021-12-10T13:00:41.396625Z 2021-12-10T13:00:41.830130Z 2021-12-10T13:00:40.441834Z 2021-12-10T13:00:37.703127Z 2021-12-10T13:00:49.134942Z 2021-12-10T13:00:35.365650Z 2021-12-10T13:00:37.484453Z 2021-12-10T13:00:23.629511Z 2021-12-10T13:00:16.978046Z 2021-12-10T13:00:22.116480Z", + "failed_job_ids": " []", + "finished": "2021-12-10T12:42:47.540633Z 2021-12-10T12:43:31.631338Z 2021-12-10T12:42:49.992395Z 2021-12-10T12:43:20.915485Z 2021-12-10T12:42:49.820445Z 2021-12-10T12:43:30.088993Z 2021-12-10T12:42:52.832477Z 2021-12-10T12:43:41.058051Z 2021-12-10T12:42:37.469098Z 2021-12-10T12:43:35.498280Z 2021-12-10T12:42:46.075701Z 2021-12-10T12:43:17.369522Z 2021-12-10T12:44:50.026672Z 2021-12-10T12:43:30.101196Z 2021-12-10T12:44:30.768121Z 2021-12-10T12:43:16.103866Z 2021-12-10T12:44:20.602121Z 2021-12-10T12:43:14.249568Z 2021-12-10T12:44:59.127291Z 2021-12-10T12:43:13.695820Z 2021-12-10T12:44:02.788987Z 2021-12-10T12:43:09.802838Z 2021-12-10T12:43:33.517999Z 2021-12-10T12:42:40.389597Z 2021-12-10T12:45:09.201299Z 2021-12-10T12:42:41.988245Z 2021-12-10T12:44:40.865550Z 2021-12-10T12:42:42.648121Z 2021-12-10T12:44:12.691459Z 2021-12-10T12:42:30.562235Z 2021-12-10T12:43:44.514313Z 2021-12-10T12:42:27.474440Z 2021-12-10T12:43:54.426422Z 2021-12-10T12:42:19.345012Z 2021-12-10T12:42:38.860724Z 2021-12-10T12:42:17.060977Z 2021-12-10T12:43:32.545250Z 2021-12-10T12:43:52.534684Z 2021-12-10T12:43:34.974764Z 2021-12-10T12:43:51.635939Z 2021-12-10T12:43:35.195221Z 2021-12-10T12:43:47.389767Z 2021-12-10T12:43:34.331259Z 2021-12-10T12:43:51.221864Z 2021-12-10T12:43:31.279975Z 2021-12-10T12:43:51.878850Z 2021-12-10T12:43:34.876160Z 2021-12-10T12:43:49.373182Z 2021-12-10T12:43:33.230796Z 2021-12-10T12:43:44.385809Z 2021-12-10T12:43:51.748364Z 2021-12-10T12:43:31.270076Z 2021-12-10T12:43:52.351367Z 2021-12-10T12:43:29.336572Z 2021-12-10T12:43:41.440859Z 2021-12-10T12:43:28.336661Z 2021-12-10T12:43:52.314173Z 2021-12-10T12:43:32.901023Z 2021-12-10T12:43:48.572559Z 2021-12-10T12:43:25.549039Z 2021-12-10T12:43:49.001322Z 2021-12-10T12:43:31.225137Z 2021-12-10T12:43:47.128001Z 2021-12-10T12:43:32.868582Z 2021-12-10T12:43:40.722098Z 2021-12-10T12:43:31.314259Z 2021-12-10T12:43:44.667864Z 2021-12-10T12:43:24.105765Z 2021-12-10T12:43:45.746405Z 2021-12-10T12:43:25.023968Z 2021-12-10T12:43:38.987055Z 2021-12-10T12:43:32.191214Z 2021-12-10T12:43:50.075787Z 2021-12-10T12:43:28.405365Z 2021-12-10T12:43:46.112668Z 2021-12-10T12:43:21.376557Z 2021-12-10T12:43:44.041889Z 2021-12-10T12:43:27.505557Z 2021-12-10T12:43:48.292741Z 2021-12-10T12:43:19.844419Z 2021-12-10T12:43:46.985371Z 2021-12-10T12:43:26.393547Z 2021-12-10T12:43:44.593346Z 2021-12-10T12:43:23.702893Z 2021-12-10T12:43:48.993447Z 2021-12-10T12:43:18.238441Z 2021-12-10T12:43:32.872104Z 2021-12-10T12:43:16.700233Z 2021-12-10T12:43:46.777347Z 2021-12-10T12:43:27.941500Z 2021-12-10T12:43:46.684654Z 2021-12-10T12:43:14.681658Z 2021-12-10T12:43:36.192907Z 2021-12-10T12:43:14.352606Z 2021-12-10T12:43:47.444900Z 2021-12-10T12:43:18.836760Z 2021-12-10T12:43:39.938401Z 2021-12-10T12:43:33.301708Z 2021-12-10T12:43:01.245242Z 2021-12-10T12:43:30.162970Z 2021-12-10T12:48:56.041659Z 2021-12-10T12:48:57.562324Z 2021-12-10T12:49:00.054200Z 2021-12-10T12:48:57.472475Z 2021-12-10T12:48:57.818539Z 2021-12-10T12:48:57.274449Z 2021-12-10T12:49:00.554299Z 2021-12-10T12:48:56.637489Z 2021-12-10T12:49:00.067491Z 2021-12-10T12:48:53.517896Z 2021-12-10T12:49:00.549470Z 2021-12-10T12:48:57.335428Z 2021-12-10T12:48:57.268867Z 2021-12-10T12:48:56.781918Z 2021-12-10T12:49:00.047716Z 2021-12-10T12:48:56.345453Z 2021-12-10T12:48:59.374705Z 2021-12-10T12:48:55.454150Z 2021-12-10T12:48:57.359329Z 2021-12-10T12:48:57.137806Z 2021-12-10T12:48:55.844050Z 2021-12-10T12:48:56.842066Z 2021-12-10T12:48:54.149329Z 2021-12-10T12:48:55.826328Z 2021-12-10T12:48:58.186586Z 2021-12-10T12:48:51.595593Z 2021-12-10T12:48:54.338552Z 2021-12-10T12:48:53.973641Z 2021-12-10T12:48:53.870858Z 2021-12-10T12:48:59.990607Z 2021-12-10T12:48:56.441904Z 2021-12-10T12:48:50.525579Z 2021-12-10T12:48:58.688753Z 2021-12-10T12:48:54.251140Z 2021-12-10T12:48:50.946129Z 2021-12-10T12:48:48.735442Z 2021-12-10T12:48:48.019630Z 2021-12-10T12:48:48.748404Z 2021-12-10T12:48:48.537769Z 2021-12-10T12:48:49.011914Z 2021-12-10T12:48:49.430703Z 2021-12-10T12:48:50.270892Z 2021-12-10T12:48:44.025624Z 2021-12-10T12:48:45.867630Z 2021-12-10T12:48:50.250609Z 2021-12-10T12:48:43.865608Z 2021-12-10T12:48:44.619813Z 2021-12-10T12:48:49.747438Z 2021-12-10T12:48:40.986426Z 2021-12-10T12:48:47.813062Z 2021-12-10T12:48:47.394353Z 2021-12-10T12:48:51.219744Z 2021-12-10T12:48:40.359536Z 2021-12-10T12:48:41.606885Z 2021-12-10T12:48:42.593445Z 2021-12-10T12:48:38.819731Z 2021-12-10T12:48:36.412751Z 2021-12-10T12:48:35.250855Z 2021-12-10T12:48:34.066800Z 2021-12-10T12:48:44.315839Z 2021-12-10T12:48:36.040532Z 2021-12-10T12:48:35.110250Z 2021-12-10T12:48:32.409432Z 2021-12-10T12:48:37.564201Z 2021-12-10T12:48:38.272007Z 2021-12-10T12:48:29.160527Z 2021-12-10T12:48:38.337059Z 2021-12-10T12:48:34.315889Z 2021-12-10T12:48:37.716912Z 2021-12-10T12:48:29.193905Z 2021-12-10T12:48:28.768691Z 2021-12-10T12:48:36.732017Z 2021-12-10T12:48:31.755060Z 2021-12-10T12:48:32.212262Z 2021-12-10T12:48:31.142346Z 2021-12-10T12:48:04.096483Z 2021-12-10T12:48:06.683039Z 2021-12-10T12:48:24.519891Z 2021-12-10T12:48:24.792265Z 2021-12-10T12:48:23.181859Z 2021-12-10T12:47:52.370160Z 2021-12-10T12:47:58.246197Z 2021-12-10T12:47:54.061219Z 2021-12-10T12:47:53.025752Z 2021-12-10T12:47:49.941031Z 2021-12-10T12:47:59.160054Z 2021-12-10T12:47:51.386959Z 2021-12-10T12:47:49.373784Z 2021-12-10T12:47:45.907306Z 2021-12-10T12:47:45.421099Z 2021-12-10T12:47:46.728559Z 2021-12-10T12:47:46.298344Z 2021-12-10T12:47:42.193457Z 2021-12-10T12:47:36.625008Z 2021-12-10T12:47:48.504466Z 2021-12-10T12:47:43.238115Z 2021-12-10T12:47:30.618748Z 2021-12-10T12:47:30.948829Z 2021-12-10T12:47:30.495141Z 2021-12-10T12:47:35.035119Z 2021-12-10T12:53:12.978986Z 2021-12-10T12:53:14.226860Z 2021-12-10T12:53:11.415476Z 2021-12-10T12:53:14.255153Z 2021-12-10T12:53:12.426938Z 2021-12-10T12:53:14.225376Z 2021-12-10T12:53:13.097175Z 2021-12-10T12:53:15.051017Z 2021-12-10T12:53:07.375981Z 2021-12-10T12:53:14.391968Z 2021-12-10T12:53:13.189030Z 2021-12-10T12:53:12.168105Z 2021-12-10T12:53:11.697638Z 2021-12-10T12:53:11.810503Z 2021-12-10T12:53:11.895143Z 2021-12-10T12:53:14.416293Z 2021-12-10T12:53:12.235442Z 2021-12-10T12:53:11.816408Z 2021-12-10T12:53:10.165234Z 2021-12-10T12:53:13.833126Z 2021-12-10T12:53:10.062983Z 2021-12-10T12:53:14.564805Z 2021-12-10T12:53:10.083344Z 2021-12-10T12:53:13.435175Z 2021-12-10T12:53:12.741366Z 2021-12-10T12:53:15.161153Z 2021-12-10T12:53:08.395272Z 2021-12-10T12:53:03.788493Z 2021-12-10T12:53:09.642958Z 2021-12-10T12:53:09.347779Z 2021-12-10T12:53:03.930222Z 2021-12-10T12:53:13.550463Z 2021-12-10T12:53:08.516470Z 2021-12-10T12:53:11.870851Z 2021-12-10T12:53:11.850535Z 2021-12-10T12:53:10.002508Z 2021-12-10T12:53:07.254479Z 2021-12-10T12:53:10.928637Z 2021-12-10T12:53:03.484112Z 2021-12-10T12:53:11.131792Z 2021-12-10T12:53:03.166003Z 2021-12-10T12:53:06.056552Z 2021-12-10T12:53:03.442393Z 2021-12-10T12:53:06.730540Z 2021-12-10T12:53:08.047002Z 2021-12-10T12:53:12.669368Z 2021-12-10T12:53:03.365563Z 2021-12-10T12:53:08.286097Z 2021-12-10T12:53:01.008113Z 2021-12-10T12:53:03.120341Z 2021-12-10T12:52:57.642210Z 2021-12-10T12:53:05.182294Z 2021-12-10T12:53:04.932396Z 2021-12-10T12:53:06.296581Z 2021-12-10T12:52:59.998505Z 2021-12-10T12:52:55.492715Z 2021-12-10T12:53:00.812078Z 2021-12-10T12:53:05.264160Z 2021-12-10T12:52:58.062420Z 2021-12-10T12:52:59.969981Z 2021-12-10T12:52:40.882530Z 2021-12-10T12:52:43.700804Z 2021-12-10T12:52:42.428553Z 2021-12-10T12:52:38.408477Z 2021-12-10T12:52:37.957047Z 2021-12-10T12:52:39.499879Z 2021-12-10T12:52:37.834040Z 2021-12-10T12:52:40.314005Z 2021-12-10T12:52:32.286624Z 2021-12-10T12:52:33.837740Z 2021-12-10T12:52:39.504206Z 2021-12-10T12:52:43.464020Z 2021-12-10T12:52:38.663449Z 2021-12-10T12:52:31.754365Z 2021-12-10T12:52:31.082504Z 2021-12-10T12:52:26.311465Z 2021-12-10T12:52:21.835804Z 2021-12-10T12:52:33.034326Z 2021-12-10T12:52:24.341315Z 2021-12-10T12:52:22.799572Z 2021-12-10T12:52:21.878914Z 2021-12-10T12:52:16.632263Z 2021-12-10T12:52:22.289481Z 2021-12-10T12:52:20.988193Z 2021-12-10T12:52:21.551728Z 2021-12-10T12:52:17.685037Z 2021-12-10T12:52:09.475837Z 2021-12-10T12:52:08.351980Z 2021-12-10T12:52:14.095630Z 2021-12-10T12:52:16.209407Z 2021-12-10T12:52:06.532257Z 2021-12-10T12:51:54.681081Z 2021-12-10T12:51:52.146608Z 2021-12-10T12:51:55.202078Z 2021-12-10T12:51:50.885584Z 2021-12-10T12:51:47.881280Z 2021-12-10T12:51:46.641999Z 2021-12-10T12:51:48.692021Z 2021-12-10T12:51:43.662504Z 2021-12-10T12:51:46.460148Z 2021-12-10T12:57:34.569529Z 2021-12-10T12:57:31.607502Z 2021-12-10T12:57:34.115023Z 2021-12-10T12:57:32.306148Z 2021-12-10T12:57:34.249638Z 2021-12-10T12:57:30.948020Z 2021-12-10T12:57:29.007551Z 2021-12-10T12:57:33.846903Z 2021-12-10T12:57:27.704763Z 2021-12-10T12:57:32.615203Z 2021-12-10T12:57:30.208769Z 2021-12-10T12:57:29.779046Z 2021-12-10T12:57:28.310140Z 2021-12-10T12:57:31.455507Z 2021-12-10T12:57:29.396003Z 2021-12-10T12:57:30.462378Z 2021-12-10T12:57:28.581612Z 2021-12-10T12:57:32.291757Z 2021-12-10T12:57:30.558206Z 2021-12-10T12:57:29.201425Z 2021-12-10T12:57:29.865826Z 2021-12-10T12:57:33.513745Z 2021-12-10T12:57:27.575564Z 2021-12-10T12:57:33.103770Z 2021-12-10T12:57:29.419448Z 2021-12-10T12:57:30.664462Z 2021-12-10T12:57:30.251157Z 2021-12-10T12:57:32.659696Z 2021-12-10T12:57:31.052571Z 2021-12-10T12:57:31.522797Z 2021-12-10T12:57:24.111119Z 2021-12-10T12:57:26.474151Z 2021-12-10T12:57:27.608125Z 2021-12-10T12:57:23.108370Z 2021-12-10T12:57:26.082498Z 2021-12-10T12:57:32.058242Z 2021-12-10T12:57:25.762186Z 2021-12-10T12:57:29.068311Z 2021-12-10T12:57:25.678259Z 2021-12-10T12:57:27.165823Z 2021-12-10T12:57:26.118054Z 2021-12-10T12:57:19.902297Z 2021-12-10T12:57:24.483545Z 2021-12-10T12:57:23.103615Z 2021-12-10T12:57:30.590195Z 2021-12-10T12:57:26.214456Z 2021-12-10T12:57:27.351612Z 2021-12-10T12:57:23.827115Z 2021-12-10T12:57:17.758770Z 2021-12-10T12:57:31.795958Z 2021-12-10T12:57:22.009970Z 2021-12-10T12:57:16.857417Z 2021-12-10T12:57:21.659158Z 2021-12-10T12:57:21.697634Z 2021-12-10T12:57:20.499908Z 2021-12-10T12:57:21.453551Z 2021-12-10T12:57:18.864504Z 2021-12-10T12:57:21.661065Z 2021-12-10T12:57:20.549550Z 2021-12-10T12:57:18.732301Z 2021-12-10T12:57:16.230455Z 2021-12-10T12:57:18.785522Z 2021-12-10T12:57:11.416364Z 2021-12-10T12:57:11.349018Z 2021-12-10T12:57:07.982405Z 2021-12-10T12:57:06.369021Z 2021-12-10T12:57:01.293341Z 2021-12-10T12:57:09.722663Z 2021-12-10T12:57:11.959578Z 2021-12-10T12:56:59.473760Z 2021-12-10T12:57:12.007324Z 2021-12-10T12:57:02.024055Z 2021-12-10T12:57:12.461309Z 2021-12-10T12:56:59.832256Z 2021-12-10T12:57:06.617618Z 2021-12-10T12:57:05.569079Z 2021-12-10T12:57:11.418581Z 2021-12-10T12:56:53.002010Z 2021-12-10T12:57:00.474114Z 2021-12-10T12:57:00.426424Z 2021-12-10T12:56:57.959265Z 2021-12-10T12:56:46.222441Z 2021-12-10T12:56:51.156042Z 2021-12-10T12:56:19.483244Z 2021-12-10T12:56:27.351680Z 2021-12-10T12:56:21.798064Z 2021-12-10T12:56:20.887494Z 2021-12-10T12:56:22.051007Z 2021-12-10T12:56:22.851509Z 2021-12-10T12:56:19.945976Z 2021-12-10T12:56:18.668954Z 2021-12-10T12:56:19.197170Z 2021-12-10T12:56:20.000285Z 2021-12-10T12:56:18.639314Z 2021-12-10T12:56:14.674133Z 2021-12-10T12:56:10.739091Z 2021-12-10T12:56:05.309444Z 2021-12-10T12:56:07.928325Z 2021-12-10T12:56:02.645269Z 2021-12-10T12:56:04.483992Z 2021-12-10T13:01:54.470701Z 2021-12-10T13:01:51.129842Z 2021-12-10T13:01:53.522411Z 2021-12-10T13:01:50.903282Z 2021-12-10T13:01:54.723835Z 2021-12-10T13:01:50.508032Z 2021-12-10T13:01:54.133313Z 2021-12-10T13:01:48.353133Z 2021-12-10T13:01:52.096239Z 2021-12-10T13:01:51.043680Z 2021-12-10T13:01:53.521889Z 2021-12-10T13:01:50.344068Z 2021-12-10T13:01:50.676538Z 2021-12-10T13:01:44.527800Z 2021-12-10T13:01:53.188131Z 2021-12-10T13:01:52.175875Z 2021-12-10T13:01:51.589491Z 2021-12-10T13:01:49.693695Z 2021-12-10T13:01:49.850751Z 2021-12-10T13:01:50.405884Z 2021-12-10T13:01:53.512966Z 2021-12-10T13:01:45.575957Z 2021-12-10T13:01:49.212062Z 2021-12-10T13:01:44.428728Z 2021-12-10T13:01:50.550579Z 2021-12-10T13:01:45.322087Z 2021-12-10T13:01:49.368522Z 2021-12-10T13:01:50.662623Z 2021-12-10T13:01:46.580858Z 2021-12-10T13:01:47.795580Z 2021-12-10T13:01:52.392038Z 2021-12-10T13:01:42.364907Z 2021-12-10T13:01:41.143079Z 2021-12-10T13:01:47.563317Z 2021-12-10T13:01:48.779010Z 2021-12-10T13:01:49.836274Z 2021-12-10T13:01:49.198047Z 2021-12-10T13:01:42.804072Z 2021-12-10T13:01:43.849079Z 2021-12-10T13:01:47.573105Z 2021-12-10T13:01:46.045569Z 2021-12-10T13:01:45.241167Z 2021-12-10T13:01:51.939510Z 2021-12-10T13:01:46.014453Z 2021-12-10T13:01:47.060613Z 2021-12-10T13:01:44.959924Z 2021-12-10T13:01:44.216586Z 2021-12-10T13:01:41.694243Z 2021-12-10T13:01:48.441783Z 2021-12-10T13:01:35.842979Z 2021-12-10T13:01:46.805207Z 2021-12-10T13:01:38.365727Z 2021-12-10T13:01:41.184941Z 2021-12-10T13:01:40.902431Z 2021-12-10T13:01:37.616443Z 2021-12-10T13:01:38.180024Z 2021-12-10T13:01:43.290114Z 2021-12-10T13:01:33.869482Z 2021-12-10T13:01:36.038408Z 2021-12-10T13:01:34.703397Z 2021-12-10T13:01:37.629583Z 2021-12-10T13:01:35.377821Z 2021-12-10T13:01:42.891721Z 2021-12-10T13:01:31.477983Z 2021-12-10T13:01:33.875134Z 2021-12-10T13:01:31.801318Z 2021-12-10T13:01:27.978321Z 2021-12-10T13:01:28.552115Z 2021-12-10T13:01:29.045270Z 2021-12-10T13:01:35.883613Z 2021-12-10T13:01:37.980580Z 2021-12-10T13:01:34.469733Z 2021-12-10T13:01:30.022692Z 2021-12-10T13:01:29.052304Z 2021-12-10T13:01:13.390586Z 2021-12-10T13:01:07.031199Z 2021-12-10T13:01:05.665580Z 2021-12-10T13:01:07.000832Z 2021-12-10T13:01:06.628642Z 2021-12-10T13:00:53.751653Z 2021-12-10T13:01:06.496933Z 2021-12-10T13:00:54.575718Z 2021-12-10T13:01:09.679314Z 2021-12-10T13:01:00.465873Z 2021-12-10T13:01:04.381914Z 2021-12-10T13:00:49.692622Z 2021-12-10T13:01:03.329323Z 2021-12-10T13:00:52.788171Z 2021-12-10T13:00:41.917538Z 2021-12-10T13:00:46.657178Z 2021-12-10T13:00:46.959138Z 2021-12-10T13:00:47.568112Z 2021-12-10T13:00:46.307102Z 2021-12-10T13:00:41.968046Z 2021-12-10T13:00:56.253551Z 2021-12-10T13:00:40.771890Z 2021-12-10T13:00:42.478313Z 2021-12-10T13:00:27.902163Z 2021-12-10T13:00:21.143227Z 2021-12-10T13:00:23.374433Z", + "finished_diff": 107.8405994, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 179.934825, + "mean": 53.591202661999965, + "min": 11.276793 + }, + "jobs_events_duration": { + "max": 6.963151, + "mean": 3.102651902000001, + "min": 0.15446 + }, + "jobs_events_lag": { + "max": -0.923817, + "mean": -4.952595832, + "min": -7.560876 + }, + "jobs_waiting": { + "max": 43.983481, + "mean": 21.451712620000006, + "min": 0.548563 + }, + "modified": "2021-12-10T12:42:13.263477Z 2021-12-10T12:42:13.118648Z 2021-12-10T12:42:13.050755Z 2021-12-10T12:42:12.933078Z 2021-12-10T12:42:12.807281Z 2021-12-10T12:42:12.698679Z 2021-12-10T12:42:12.586729Z 2021-12-10T12:42:12.445116Z 2021-12-10T12:42:12.310263Z 2021-12-10T12:42:12.182507Z 2021-12-10T12:42:12.044739Z 2021-12-10T12:42:11.994578Z 2021-12-10T12:42:11.844473Z 2021-12-10T12:42:11.752330Z 2021-12-10T12:42:11.619681Z 2021-12-10T12:42:11.505341Z 2021-12-10T12:42:11.405396Z 2021-12-10T12:42:11.322359Z 2021-12-10T12:42:11.245293Z 2021-12-10T12:42:11.146728Z 2021-12-10T12:42:11.032540Z 2021-12-10T12:42:10.932288Z 2021-12-10T12:42:07.832235Z 2021-12-10T12:42:07.765201Z 2021-12-10T12:42:07.679030Z 2021-12-10T12:42:07.617407Z 2021-12-10T12:42:07.532316Z 2021-12-10T12:42:07.335094Z 2021-12-10T12:42:07.173207Z 2021-12-10T12:42:07.065262Z 2021-12-10T12:42:06.052554Z 2021-12-10T12:42:05.973798Z 2021-12-10T12:42:05.363386Z 2021-12-10T12:42:04.998286Z 2021-12-10T12:42:04.541646Z 2021-12-10T12:42:03.975199Z 2021-12-10T12:42:42.287990Z 2021-12-10T12:42:42.225225Z 2021-12-10T12:42:42.170860Z 2021-12-10T12:42:42.094422Z 2021-12-10T12:42:42.027485Z 2021-12-10T12:42:41.977575Z 2021-12-10T12:42:41.921503Z 2021-12-10T12:42:41.855425Z 2021-12-10T12:42:41.780536Z 2021-12-10T12:42:41.709802Z 2021-12-10T12:42:41.600940Z 2021-12-10T12:42:41.447867Z 2021-12-10T12:42:41.270674Z 2021-12-10T12:42:41.130682Z 2021-12-10T12:42:40.930773Z 2021-12-10T12:42:23.562865Z 2021-12-10T12:42:23.518728Z 2021-12-10T12:42:23.469436Z 2021-12-10T12:42:23.416437Z 2021-12-10T12:42:23.329641Z 2021-12-10T12:42:23.251250Z 2021-12-10T12:42:23.166549Z 2021-12-10T12:42:23.111680Z 2021-12-10T12:42:23.058414Z 2021-12-10T12:42:23.008472Z 2021-12-10T12:42:22.950523Z 2021-12-10T12:42:22.865518Z 2021-12-10T12:42:22.767623Z 2021-12-10T12:42:22.692815Z 2021-12-10T12:42:22.629743Z 2021-12-10T12:42:22.554157Z 2021-12-10T12:42:22.490614Z 2021-12-10T12:42:22.412936Z 2021-12-10T12:42:22.322516Z 2021-12-10T12:42:22.221581Z 2021-12-10T12:42:22.132696Z 2021-12-10T12:42:22.048473Z 2021-12-10T12:42:21.965504Z 2021-12-10T12:42:21.887155Z 2021-12-10T12:42:21.847446Z 2021-12-10T12:42:21.802726Z 2021-12-10T12:42:21.762646Z 2021-12-10T12:42:21.705732Z 2021-12-10T12:42:21.625657Z 2021-12-10T12:42:21.562492Z 2021-12-10T12:42:21.512105Z 2021-12-10T12:42:21.452935Z 2021-12-10T12:42:21.410660Z 2021-12-10T12:42:21.362927Z 2021-12-10T12:42:21.311536Z 2021-12-10T12:42:21.269517Z 2021-12-10T12:42:21.227677Z 2021-12-10T12:42:21.180444Z 2021-12-10T12:42:21.117727Z 2021-12-10T12:42:21.070729Z 2021-12-10T12:42:21.029864Z 2021-12-10T12:42:20.980702Z 2021-12-10T12:42:20.917921Z 2021-12-10T12:42:20.872473Z 2021-12-10T12:42:20.815511Z 2021-12-10T12:42:20.724084Z 2021-12-10T12:42:13.641306Z 2021-12-10T12:42:13.565564Z 2021-12-10T12:42:13.469299Z 2021-12-10T12:47:57.123710Z 2021-12-10T12:47:57.010764Z 2021-12-10T12:47:56.915628Z 2021-12-10T12:47:56.750598Z 2021-12-10T12:47:56.619604Z 2021-12-10T12:47:56.491523Z 2021-12-10T12:47:56.399559Z 2021-12-10T12:47:56.303604Z 2021-12-10T12:47:56.215489Z 2021-12-10T12:47:56.128492Z 2021-12-10T12:47:56.023788Z 2021-12-10T12:47:55.888613Z 2021-12-10T12:47:55.783791Z 2021-12-10T12:47:55.671852Z 2021-12-10T12:47:55.565107Z 2021-12-10T12:47:55.452064Z 2021-12-10T12:47:55.389721Z 2021-12-10T12:47:55.337599Z 2021-12-10T12:47:55.266715Z 2021-12-10T12:47:55.192579Z 2021-12-10T12:47:55.131592Z 2021-12-10T12:47:55.069570Z 2021-12-10T12:47:55.009686Z 2021-12-10T12:47:54.940745Z 2021-12-10T12:47:54.861551Z 2021-12-10T12:47:54.795617Z 2021-12-10T12:47:54.719595Z 2021-12-10T12:47:54.655695Z 2021-12-10T12:47:54.537321Z 2021-12-10T12:47:54.588304Z 2021-12-10T12:47:54.489337Z 2021-12-10T12:47:37.180403Z 2021-12-10T12:47:54.374620Z 2021-12-10T12:47:54.268703Z 2021-12-10T12:47:37.147995Z 2021-12-10T12:47:37.059328Z 2021-12-10T12:47:36.982322Z 2021-12-10T12:47:36.872087Z 2021-12-10T12:47:36.758266Z 2021-12-10T12:47:36.634283Z 2021-12-10T12:47:36.529287Z 2021-12-10T12:47:36.438135Z 2021-12-10T12:47:36.348230Z 2021-12-10T12:47:36.240158Z 2021-12-10T12:47:36.147009Z 2021-12-10T12:47:36.062222Z 2021-12-10T12:47:35.953161Z 2021-12-10T12:47:35.847072Z 2021-12-10T12:47:35.746152Z 2021-12-10T12:47:35.632097Z 2021-12-10T12:47:35.509358Z 2021-12-10T12:47:35.384888Z 2021-12-10T12:47:35.237156Z 2021-12-10T12:47:35.174668Z 2021-12-10T12:47:35.110064Z 2021-12-10T12:47:35.071562Z 2021-12-10T12:47:35.028290Z 2021-12-10T12:47:34.969414Z 2021-12-10T12:47:34.904190Z 2021-12-10T12:47:34.849213Z 2021-12-10T12:47:34.787157Z 2021-12-10T12:47:34.720233Z 2021-12-10T12:47:34.657245Z 2021-12-10T12:47:34.556197Z 2021-12-10T12:47:34.452183Z 2021-12-10T12:47:34.343350Z 2021-12-10T12:47:34.136345Z 2021-12-10T12:47:34.026920Z 2021-12-10T12:47:33.907198Z 2021-12-10T12:47:33.857457Z 2021-12-10T12:47:33.774078Z 2021-12-10T12:47:33.671081Z 2021-12-10T12:47:33.594461Z 2021-12-10T12:47:33.530716Z 2021-12-10T12:47:33.340221Z 2021-12-10T12:47:19.355281Z 2021-12-10T12:47:19.255947Z 2021-12-10T12:47:33.145307Z 2021-12-10T12:47:32.977726Z 2021-12-10T12:47:32.757850Z 2021-12-10T12:47:19.157137Z 2021-12-10T12:47:18.995529Z 2021-12-10T12:47:18.899425Z 2021-12-10T12:47:18.839305Z 2021-12-10T12:47:18.799178Z 2021-12-10T12:47:18.763872Z 2021-12-10T12:47:18.708221Z 2021-12-10T12:47:18.567928Z 2021-12-10T12:47:18.487091Z 2021-12-10T12:47:18.444441Z 2021-12-10T12:47:18.335850Z 2021-12-10T12:47:18.235547Z 2021-12-10T12:47:18.137040Z 2021-12-10T12:47:16.314018Z 2021-12-10T12:47:18.048584Z 2021-12-10T12:47:17.929844Z 2021-12-10T12:47:16.240891Z 2021-12-10T12:47:16.181724Z 2021-12-10T12:47:16.132655Z 2021-12-10T12:47:16.070298Z 2021-12-10T12:52:00.972916Z 2021-12-10T12:52:00.924108Z 2021-12-10T12:52:00.864614Z 2021-12-10T12:52:00.759755Z 2021-12-10T12:52:00.642194Z 2021-12-10T12:52:00.563154Z 2021-12-10T12:52:00.464035Z 2021-12-10T12:52:00.373230Z 2021-12-10T12:52:00.287199Z 2021-12-10T12:52:00.188187Z 2021-12-10T12:52:00.109326Z 2021-12-10T12:52:00.023278Z 2021-12-10T12:51:59.937333Z 2021-12-10T12:51:59.854044Z 2021-12-10T12:51:59.769445Z 2021-12-10T12:51:59.680095Z 2021-12-10T12:51:59.591234Z 2021-12-10T12:51:59.510086Z 2021-12-10T12:51:59.420285Z 2021-12-10T12:51:59.326130Z 2021-12-10T12:51:59.186326Z 2021-12-10T12:51:59.053195Z 2021-12-10T12:51:58.918048Z 2021-12-10T12:51:58.853983Z 2021-12-10T12:51:58.797998Z 2021-12-10T12:51:58.744985Z 2021-12-10T12:51:58.675018Z 2021-12-10T12:51:58.597099Z 2021-12-10T12:51:58.522564Z 2021-12-10T12:51:58.465320Z 2021-12-10T12:51:58.415288Z 2021-12-10T12:51:58.361151Z 2021-12-10T12:51:58.309085Z 2021-12-10T12:51:58.254075Z 2021-12-10T12:51:58.195122Z 2021-12-10T12:51:58.146093Z 2021-12-10T12:51:58.095201Z 2021-12-10T12:51:58.037361Z 2021-12-10T12:51:57.987070Z 2021-12-10T12:51:57.934028Z 2021-12-10T12:51:57.879040Z 2021-12-10T12:51:57.823321Z 2021-12-10T12:51:57.731115Z 2021-12-10T12:51:57.785142Z 2021-12-10T12:51:57.675008Z 2021-12-10T12:51:57.619160Z 2021-12-10T12:51:57.573004Z 2021-12-10T12:51:57.523079Z 2021-12-10T12:51:57.470217Z 2021-12-10T12:51:57.424125Z 2021-12-10T12:51:57.363120Z 2021-12-10T12:51:57.283658Z 2021-12-10T12:51:57.202195Z 2021-12-10T12:51:57.125232Z 2021-12-10T12:51:57.017206Z 2021-12-10T12:51:56.935199Z 2021-12-10T12:51:56.835113Z 2021-12-10T12:51:56.735107Z 2021-12-10T12:51:56.612344Z 2021-12-10T12:51:56.503277Z 2021-12-10T12:51:44.616580Z 2021-12-10T12:51:44.468360Z 2021-12-10T12:51:44.222462Z 2021-12-10T12:51:44.014686Z 2021-12-10T12:51:43.872483Z 2021-12-10T12:51:43.735005Z 2021-12-10T12:51:43.621256Z 2021-12-10T12:51:43.369188Z 2021-12-10T12:51:43.190770Z 2021-12-10T12:51:42.937202Z 2021-12-10T12:51:42.812737Z 2021-12-10T12:51:42.736099Z 2021-12-10T12:51:42.665259Z 2021-12-10T12:51:42.594458Z 2021-12-10T12:51:42.234371Z 2021-12-10T12:51:41.701793Z 2021-12-10T12:51:41.329714Z 2021-12-10T12:51:40.978131Z 2021-12-10T12:51:40.621277Z 2021-12-10T12:51:40.341076Z 2021-12-10T12:51:40.164624Z 2021-12-10T12:51:40.053407Z 2021-12-10T12:51:39.952829Z 2021-12-10T12:51:39.849770Z 2021-12-10T12:51:39.738735Z 2021-12-10T12:51:39.571537Z 2021-12-10T12:51:39.329962Z 2021-12-10T12:51:39.029815Z 2021-12-10T12:51:38.670168Z 2021-12-10T12:51:38.317682Z 2021-12-10T12:51:38.031182Z 2021-12-10T12:51:33.289018Z 2021-12-10T12:51:33.058210Z 2021-12-10T12:51:32.917244Z 2021-12-10T12:51:32.772356Z 2021-12-10T12:51:31.660466Z 2021-12-10T12:51:31.625992Z 2021-12-10T12:51:31.588837Z 2021-12-10T12:51:31.223676Z 2021-12-10T12:51:31.188160Z 2021-12-10T12:56:44.164575Z 2021-12-10T12:56:44.031747Z 2021-12-10T12:56:43.885559Z 2021-12-10T12:56:43.729687Z 2021-12-10T12:56:43.572873Z 2021-12-10T12:56:07.979951Z 2021-12-10T12:56:07.943594Z 2021-12-10T12:56:07.890004Z 2021-12-10T12:56:07.819362Z 2021-12-10T12:56:07.760598Z 2021-12-10T12:56:07.718336Z 2021-12-10T12:56:07.683558Z 2021-12-10T12:56:07.649744Z 2021-12-10T12:56:07.616735Z 2021-12-10T12:56:07.580899Z 2021-12-10T12:56:07.541467Z 2021-12-10T12:56:07.501923Z 2021-12-10T12:56:07.466956Z 2021-12-10T12:56:07.430102Z 2021-12-10T12:56:07.394681Z 2021-12-10T12:56:07.357921Z 2021-12-10T12:56:07.318845Z 2021-12-10T12:56:07.285022Z 2021-12-10T12:56:07.247826Z 2021-12-10T12:56:07.208421Z 2021-12-10T12:56:07.169394Z 2021-12-10T12:56:07.134125Z 2021-12-10T12:56:07.100882Z 2021-12-10T12:56:07.065462Z 2021-12-10T12:56:07.031081Z 2021-12-10T12:56:06.964763Z 2021-12-10T12:56:06.997247Z 2021-12-10T12:56:06.932040Z 2021-12-10T12:56:06.895990Z 2021-12-10T12:56:06.863283Z 2021-12-10T12:56:06.825208Z 2021-12-10T12:56:06.790550Z 2021-12-10T12:56:06.757776Z 2021-12-10T12:56:06.719306Z 2021-12-10T12:56:06.682153Z 2021-12-10T12:56:06.647709Z 2021-12-10T12:56:06.613537Z 2021-12-10T12:56:06.579591Z 2021-12-10T12:56:06.500707Z 2021-12-10T12:56:06.539866Z 2021-12-10T12:56:06.467714Z 2021-12-10T12:56:06.421420Z 2021-12-10T12:56:06.369481Z 2021-12-10T12:56:06.312458Z 2021-12-10T12:56:06.260716Z 2021-12-10T12:56:06.209544Z 2021-12-10T12:56:06.159467Z 2021-12-10T12:56:06.108775Z 2021-12-10T12:56:06.039755Z 2021-12-10T12:56:05.975686Z 2021-12-10T12:56:05.920815Z 2021-12-10T12:56:05.857129Z 2021-12-10T12:56:05.793708Z 2021-12-10T12:56:05.725547Z 2021-12-10T12:56:05.651872Z 2021-12-10T12:56:05.577069Z 2021-12-10T12:56:05.524007Z 2021-12-10T12:56:05.461544Z 2021-12-10T12:56:05.395655Z 2021-12-10T12:56:05.335895Z 2021-12-10T12:56:05.295246Z 2021-12-10T12:56:05.249569Z 2021-12-10T12:56:05.212985Z 2021-12-10T12:56:05.130727Z 2021-12-10T12:56:04.999515Z 2021-12-10T12:56:05.071622Z 2021-12-10T12:56:04.906570Z 2021-12-10T12:56:04.792724Z 2021-12-10T12:56:04.677611Z 2021-12-10T12:56:04.562574Z 2021-12-10T12:56:04.470332Z 2021-12-10T12:56:04.423706Z 2021-12-10T12:56:04.362530Z 2021-12-10T12:56:04.308361Z 2021-12-10T12:56:04.251714Z 2021-12-10T12:56:04.193723Z 2021-12-10T12:56:04.147230Z 2021-12-10T12:56:04.108729Z 2021-12-10T12:55:54.584470Z 2021-12-10T12:55:54.372764Z 2021-12-10T12:55:54.148768Z 2021-12-10T12:55:53.878418Z 2021-12-10T12:55:53.757786Z 2021-12-10T12:55:53.598608Z 2021-12-10T12:55:53.457257Z 2021-12-10T12:55:53.341483Z 2021-12-10T12:55:53.243314Z 2021-12-10T12:55:53.126565Z 2021-12-10T12:55:53.033433Z 2021-12-10T12:55:52.917421Z 2021-12-10T12:55:50.677313Z 2021-12-10T12:55:50.583089Z 2021-12-10T12:55:50.520415Z 2021-12-10T12:55:49.875877Z 2021-12-10T12:55:49.844060Z 2021-12-10T13:00:30.188578Z 2021-12-10T13:00:30.100691Z 2021-12-10T13:00:30.011715Z 2021-12-10T13:00:29.918609Z 2021-12-10T13:00:29.826810Z 2021-12-10T13:00:29.743534Z 2021-12-10T13:00:29.651690Z 2021-12-10T13:00:29.563450Z 2021-12-10T13:00:29.491287Z 2021-12-10T13:00:29.447598Z 2021-12-10T13:00:29.410519Z 2021-12-10T13:00:29.354619Z 2021-12-10T13:00:29.300479Z 2021-12-10T13:00:29.240756Z 2021-12-10T13:00:29.182483Z 2021-12-10T13:00:29.132960Z 2021-12-10T13:00:29.077724Z 2021-12-10T13:00:29.018636Z 2021-12-10T13:00:28.964515Z 2021-12-10T13:00:28.907558Z 2021-12-10T13:00:28.856744Z 2021-12-10T13:00:28.801660Z 2021-12-10T13:00:28.748516Z 2021-12-10T13:00:28.690505Z 2021-12-10T13:00:28.632463Z 2021-12-10T13:00:28.578459Z 2021-12-10T13:00:28.455498Z 2021-12-10T13:00:28.517586Z 2021-12-10T13:00:28.405465Z 2021-12-10T13:00:28.351500Z 2021-12-10T13:00:28.304722Z 2021-12-10T13:00:28.245583Z 2021-12-10T13:00:28.181623Z 2021-12-10T13:00:28.133433Z 2021-12-10T13:00:28.091763Z 2021-12-10T13:00:28.034773Z 2021-12-10T13:00:27.981507Z 2021-12-10T13:00:27.930636Z 2021-12-10T13:00:27.882466Z 2021-12-10T13:00:27.829668Z 2021-12-10T13:00:27.781562Z 2021-12-10T13:00:27.727479Z 2021-12-10T13:00:27.682746Z 2021-12-10T13:00:27.630134Z 2021-12-10T13:00:27.570291Z 2021-12-10T13:00:27.517241Z 2021-12-10T13:00:27.467993Z 2021-12-10T13:00:27.416616Z 2021-12-10T13:00:27.369604Z 2021-12-10T13:00:27.326750Z 2021-12-10T13:00:27.282624Z 2021-12-10T13:00:27.233700Z 2021-12-10T13:00:27.185586Z 2021-12-10T13:00:27.140317Z 2021-12-10T13:00:27.086874Z 2021-12-10T13:00:27.032517Z 2021-12-10T13:00:26.968933Z 2021-12-10T13:00:26.917654Z 2021-12-10T13:00:26.872721Z 2021-12-10T13:00:26.827561Z 2021-12-10T13:00:26.773515Z 2021-12-10T13:00:26.727467Z 2021-12-10T13:00:26.689857Z 2021-12-10T13:00:26.650330Z 2021-12-10T13:00:26.585831Z 2021-12-10T13:00:26.535738Z 2021-12-10T13:00:26.486631Z 2021-12-10T13:00:26.438994Z 2021-12-10T13:00:26.401626Z 2021-12-10T13:00:26.366602Z 2021-12-10T13:00:26.317786Z 2021-12-10T13:00:26.262121Z 2021-12-10T13:00:26.201308Z 2021-12-10T13:00:26.121942Z 2021-12-10T13:00:18.251651Z 2021-12-10T13:00:18.204925Z 2021-12-10T13:00:18.157202Z 2021-12-10T13:00:18.056280Z 2021-12-10T13:00:17.829754Z 2021-12-10T13:00:17.397731Z 2021-12-10T13:00:17.609039Z 2021-12-10T13:00:17.179643Z 2021-12-10T13:00:17.007215Z 2021-12-10T13:00:16.813056Z 2021-12-10T13:00:16.613748Z 2021-12-10T13:00:16.411155Z 2021-12-10T13:00:16.336081Z 2021-12-10T13:00:16.127967Z 2021-12-10T13:00:15.808708Z 2021-12-10T13:00:15.506146Z 2021-12-10T13:00:15.151253Z 2021-12-10T13:00:14.843585Z 2021-12-10T13:00:14.651575Z 2021-12-10T13:00:14.559071Z 2021-12-10T13:00:14.412974Z 2021-12-10T13:00:14.257226Z 2021-12-10T13:00:14.075086Z 2021-12-10T13:00:10.484212Z 2021-12-10T13:00:09.578989Z 2021-12-10T13:00:09.541738Z", + "modified_diff": 36.8236628, + "started": "2021-12-10T12:42:17.935275Z 2021-12-10T12:42:18.379224Z 2021-12-10T12:42:17.500043Z 2021-12-10T12:42:17.710304Z 2021-12-10T12:42:17.080059Z 2021-12-10T12:42:17.439723Z 2021-12-10T12:42:16.813983Z 2021-12-10T12:42:17.217422Z 2021-12-10T12:42:16.626088Z 2021-12-10T12:42:17.179762Z 2021-12-10T12:42:16.438023Z 2021-12-10T12:42:16.757351Z 2021-12-10T12:42:15.900296Z 2021-12-10T12:42:16.605545Z 2021-12-10T12:42:15.530797Z 2021-12-10T12:42:15.541995Z 2021-12-10T12:42:14.893284Z 2021-12-10T12:42:15.095178Z 2021-12-10T12:42:14.625520Z 2021-12-10T12:42:14.519097Z 2021-12-10T12:42:14.215075Z 2021-12-10T12:42:14.227750Z 2021-12-10T12:42:09.822197Z 2021-12-10T12:42:09.499921Z 2021-12-10T12:42:09.266474Z 2021-12-10T12:42:09.223014Z 2021-12-10T12:42:08.895739Z 2021-12-10T12:42:08.652481Z 2021-12-10T12:42:08.451003Z 2021-12-10T12:42:08.365481Z 2021-12-10T12:42:06.423212Z 2021-12-10T12:42:06.359971Z 2021-12-10T12:42:05.586768Z 2021-12-10T12:42:05.220729Z 2021-12-10T12:42:04.739338Z 2021-12-10T12:42:04.126855Z 2021-12-10T12:42:46.766867Z 2021-12-10T12:42:46.780038Z 2021-12-10T12:42:46.155512Z 2021-12-10T12:42:45.654885Z 2021-12-10T12:42:45.451371Z 2021-12-10T12:42:45.109362Z 2021-12-10T12:42:45.162591Z 2021-12-10T12:42:44.564877Z 2021-12-10T12:42:44.222558Z 2021-12-10T12:42:44.861605Z 2021-12-10T12:42:43.568245Z 2021-12-10T12:42:43.951984Z 2021-12-10T12:42:43.198473Z 2021-12-10T12:42:43.103232Z 2021-12-10T12:42:42.917570Z 2021-12-10T12:42:38.822233Z 2021-12-10T12:42:40.275057Z 2021-12-10T12:42:37.722464Z 2021-12-10T12:42:40.244295Z 2021-12-10T12:42:37.645834Z 2021-12-10T12:42:38.891546Z 2021-12-10T12:42:36.458555Z 2021-12-10T12:42:37.910311Z 2021-12-10T12:42:35.754390Z 2021-12-10T12:42:36.682692Z 2021-12-10T12:42:35.312551Z 2021-12-10T12:42:35.740906Z 2021-12-10T12:42:34.741084Z 2021-12-10T12:42:34.814147Z 2021-12-10T12:42:34.289346Z 2021-12-10T12:42:34.435675Z 2021-12-10T12:42:33.268645Z 2021-12-10T12:42:32.760796Z 2021-12-10T12:42:32.417591Z 2021-12-10T12:42:31.586994Z 2021-12-10T12:42:31.309711Z 2021-12-10T12:42:31.554074Z 2021-12-10T12:42:30.732470Z 2021-12-10T12:42:30.676033Z 2021-12-10T12:42:30.082484Z 2021-12-10T12:42:30.891007Z 2021-12-10T12:42:29.417428Z 2021-12-10T12:42:31.145982Z 2021-12-10T12:42:28.799404Z 2021-12-10T12:42:30.210656Z 2021-12-10T12:42:27.968358Z 2021-12-10T12:42:29.327128Z 2021-12-10T12:42:27.187229Z 2021-12-10T12:42:28.676661Z 2021-12-10T12:42:26.684626Z 2021-12-10T12:42:27.058916Z 2021-12-10T12:42:26.277535Z 2021-12-10T12:42:27.095300Z 2021-12-10T12:42:25.673019Z 2021-12-10T12:42:25.935459Z 2021-12-10T12:42:24.944758Z 2021-12-10T12:42:24.872244Z 2021-12-10T12:42:24.456465Z 2021-12-10T12:42:25.584888Z 2021-12-10T12:42:24.043402Z 2021-12-10T12:42:25.769880Z 2021-12-10T12:42:19.251780Z 2021-12-10T12:42:18.610759Z 2021-12-10T12:42:18.904377Z 2021-12-10T12:48:07.895844Z 2021-12-10T12:48:07.505926Z 2021-12-10T12:48:07.379912Z 2021-12-10T12:48:07.083668Z 2021-12-10T12:48:06.625876Z 2021-12-10T12:48:06.208125Z 2021-12-10T12:48:05.544471Z 2021-12-10T12:48:05.039997Z 2021-12-10T12:48:04.715530Z 2021-12-10T12:48:04.282241Z 2021-12-10T12:48:04.032050Z 2021-12-10T12:48:03.605090Z 2021-12-10T12:48:03.534407Z 2021-12-10T12:48:02.987029Z 2021-12-10T12:48:02.676361Z 2021-12-10T12:48:02.488478Z 2021-12-10T12:48:02.376184Z 2021-12-10T12:48:01.686215Z 2021-12-10T12:48:01.822871Z 2021-12-10T12:48:01.397066Z 2021-12-10T12:48:01.090041Z 2021-12-10T12:48:00.802514Z 2021-12-10T12:48:00.407684Z 2021-12-10T12:48:00.233900Z 2021-12-10T12:48:00.051330Z 2021-12-10T12:47:59.900376Z 2021-12-10T12:47:59.781114Z 2021-12-10T12:47:59.039145Z 2021-12-10T12:47:58.426760Z 2021-12-10T12:47:59.310749Z 2021-12-10T12:47:58.351526Z 2021-12-10T12:47:52.285269Z 2021-12-10T12:47:57.932161Z 2021-12-10T12:47:57.696659Z 2021-12-10T12:47:51.822744Z 2021-12-10T12:47:51.606358Z 2021-12-10T12:47:51.081811Z 2021-12-10T12:47:50.819769Z 2021-12-10T12:47:50.742904Z 2021-12-10T12:47:50.166604Z 2021-12-10T12:47:49.895496Z 2021-12-10T12:47:49.505404Z 2021-12-10T12:47:48.746466Z 2021-12-10T12:47:48.502151Z 2021-12-10T12:47:48.103576Z 2021-12-10T12:47:48.258251Z 2021-12-10T12:47:47.485182Z 2021-12-10T12:47:46.965032Z 2021-12-10T12:47:46.964615Z 2021-12-10T12:47:46.608612Z 2021-12-10T12:47:46.170418Z 2021-12-10T12:47:46.115026Z 2021-12-10T12:47:45.568767Z 2021-12-10T12:47:45.365026Z 2021-12-10T12:47:44.935006Z 2021-12-10T12:47:44.653130Z 2021-12-10T12:47:44.284345Z 2021-12-10T12:47:44.186494Z 2021-12-10T12:47:43.518286Z 2021-12-10T12:47:43.219512Z 2021-12-10T12:47:42.553902Z 2021-12-10T12:47:42.193444Z 2021-12-10T12:47:41.664584Z 2021-12-10T12:47:41.565845Z 2021-12-10T12:47:41.314335Z 2021-12-10T12:47:40.682131Z 2021-12-10T12:47:40.593650Z 2021-12-10T12:47:40.086340Z 2021-12-10T12:47:39.443555Z 2021-12-10T12:47:39.510976Z 2021-12-10T12:47:39.350551Z 2021-12-10T12:47:38.859035Z 2021-12-10T12:47:38.612558Z 2021-12-10T12:47:38.217327Z 2021-12-10T12:47:38.194927Z 2021-12-10T12:47:25.995037Z 2021-12-10T12:47:25.769047Z 2021-12-10T12:47:37.891004Z 2021-12-10T12:47:37.787961Z 2021-12-10T12:47:37.537883Z 2021-12-10T12:47:24.301601Z 2021-12-10T12:47:24.357120Z 2021-12-10T12:47:23.650933Z 2021-12-10T12:47:23.621058Z 2021-12-10T12:47:22.625556Z 2021-12-10T12:47:23.205357Z 2021-12-10T12:47:22.336570Z 2021-12-10T12:47:23.014820Z 2021-12-10T12:47:21.632511Z 2021-12-10T12:47:21.087390Z 2021-12-10T12:47:20.882900Z 2021-12-10T12:47:21.358576Z 2021-12-10T12:47:20.502451Z 2021-12-10T12:47:17.426030Z 2021-12-10T12:47:20.468831Z 2021-12-10T12:47:19.977792Z 2021-12-10T12:47:17.201866Z 2021-12-10T12:47:17.072669Z 2021-12-10T12:47:16.853170Z 2021-12-10T12:47:16.838040Z 2021-12-10T12:52:24.178969Z 2021-12-10T12:52:23.492523Z 2021-12-10T12:52:22.862464Z 2021-12-10T12:52:22.664246Z 2021-12-10T12:52:22.552937Z 2021-12-10T12:52:22.480703Z 2021-12-10T12:52:22.122579Z 2021-12-10T12:52:21.794353Z 2021-12-10T12:52:20.885494Z 2021-12-10T12:52:20.158086Z 2021-12-10T12:52:19.976052Z 2021-12-10T12:52:19.550527Z 2021-12-10T12:52:19.064331Z 2021-12-10T12:52:18.669087Z 2021-12-10T12:52:18.481164Z 2021-12-10T12:52:18.097061Z 2021-12-10T12:52:17.136324Z 2021-12-10T12:52:16.822977Z 2021-12-10T12:52:16.541536Z 2021-12-10T12:52:16.649215Z 2021-12-10T12:52:15.817173Z 2021-12-10T12:52:15.743119Z 2021-12-10T12:52:15.079278Z 2021-12-10T12:52:14.755249Z 2021-12-10T12:52:14.504567Z 2021-12-10T12:52:14.022311Z 2021-12-10T12:52:13.618101Z 2021-12-10T12:52:13.415535Z 2021-12-10T12:52:12.683625Z 2021-12-10T12:52:12.237994Z 2021-12-10T12:52:11.879495Z 2021-12-10T12:52:11.621970Z 2021-12-10T12:52:11.182576Z 2021-12-10T12:52:10.739141Z 2021-12-10T12:52:10.223327Z 2021-12-10T12:52:10.237622Z 2021-12-10T12:52:09.139509Z 2021-12-10T12:52:08.650234Z 2021-12-10T12:52:08.365200Z 2021-12-10T12:52:08.068952Z 2021-12-10T12:52:07.595792Z 2021-12-10T12:52:07.146792Z 2021-12-10T12:52:06.605261Z 2021-12-10T12:52:06.804671Z 2021-12-10T12:52:06.522998Z 2021-12-10T12:52:06.120036Z 2021-12-10T12:52:05.804549Z 2021-12-10T12:52:05.717057Z 2021-12-10T12:52:04.895770Z 2021-12-10T12:52:04.852559Z 2021-12-10T12:52:04.193494Z 2021-12-10T12:52:03.660982Z 2021-12-10T12:52:03.311404Z 2021-12-10T12:52:02.883611Z 2021-12-10T12:52:02.508704Z 2021-12-10T12:52:02.182748Z 2021-12-10T12:52:01.675368Z 2021-12-10T12:52:02.091028Z 2021-12-10T12:52:01.432400Z 2021-12-10T12:52:02.026138Z 2021-12-10T12:51:53.338445Z 2021-12-10T12:51:52.841023Z 2021-12-10T12:51:52.522853Z 2021-12-10T12:51:52.254002Z 2021-12-10T12:51:51.963902Z 2021-12-10T12:51:51.730453Z 2021-12-10T12:51:51.452837Z 2021-12-10T12:51:51.024082Z 2021-12-10T12:51:50.833633Z 2021-12-10T12:51:50.328103Z 2021-12-10T12:51:50.141274Z 2021-12-10T12:51:49.885929Z 2021-12-10T12:51:49.673706Z 2021-12-10T12:51:49.378379Z 2021-12-10T12:51:48.903467Z 2021-12-10T12:51:48.691042Z 2021-12-10T12:51:48.526971Z 2021-12-10T12:51:48.241066Z 2021-12-10T12:51:48.153714Z 2021-12-10T12:51:47.748077Z 2021-12-10T12:51:47.547012Z 2021-12-10T12:51:47.310689Z 2021-12-10T12:51:46.893494Z 2021-12-10T12:51:46.545448Z 2021-12-10T12:51:46.315156Z 2021-12-10T12:51:46.036800Z 2021-12-10T12:51:45.811092Z 2021-12-10T12:51:45.690428Z 2021-12-10T12:51:45.453278Z 2021-12-10T12:51:45.278278Z 2021-12-10T12:51:45.156458Z 2021-12-10T12:51:35.963109Z 2021-12-10T12:51:35.148870Z 2021-12-10T12:51:35.069286Z 2021-12-10T12:51:34.055808Z 2021-12-10T12:51:32.224141Z 2021-12-10T12:51:31.998081Z 2021-12-10T12:51:31.945291Z 2021-12-10T12:51:31.477560Z 2021-12-10T12:51:31.431337Z 2021-12-10T12:56:45.460193Z 2021-12-10T12:56:45.187241Z 2021-12-10T12:56:45.007238Z 2021-12-10T12:56:44.707128Z 2021-12-10T12:56:44.554175Z 2021-12-10T12:56:33.876764Z 2021-12-10T12:56:33.555551Z 2021-12-10T12:56:33.538168Z 2021-12-10T12:56:33.002247Z 2021-12-10T12:56:32.594375Z 2021-12-10T12:56:32.364788Z 2021-12-10T12:56:32.026339Z 2021-12-10T12:56:31.548825Z 2021-12-10T12:56:31.176089Z 2021-12-10T12:56:30.772152Z 2021-12-10T12:56:30.422415Z 2021-12-10T12:56:29.991597Z 2021-12-10T12:56:29.947148Z 2021-12-10T12:56:29.649753Z 2021-12-10T12:56:29.110039Z 2021-12-10T12:56:28.697512Z 2021-12-10T12:56:28.610904Z 2021-12-10T12:56:28.224959Z 2021-12-10T12:56:27.877515Z 2021-12-10T12:56:27.805579Z 2021-12-10T12:56:27.268746Z 2021-12-10T12:56:27.030309Z 2021-12-10T12:56:26.395095Z 2021-12-10T12:56:25.955416Z 2021-12-10T12:56:25.371373Z 2021-12-10T12:56:24.834108Z 2021-12-10T12:56:25.268932Z 2021-12-10T12:56:24.609136Z 2021-12-10T12:56:24.147864Z 2021-12-10T12:56:23.739521Z 2021-12-10T12:56:23.496998Z 2021-12-10T12:56:23.633191Z 2021-12-10T12:56:22.307846Z 2021-12-10T12:56:21.815666Z 2021-12-10T12:56:21.327326Z 2021-12-10T12:56:20.682627Z 2021-12-10T12:56:20.237353Z 2021-12-10T12:56:20.448633Z 2021-12-10T12:56:19.396118Z 2021-12-10T12:56:19.838188Z 2021-12-10T12:56:19.049807Z 2021-12-10T12:56:18.951899Z 2021-12-10T12:56:18.591150Z 2021-12-10T12:56:18.347954Z 2021-12-10T12:56:18.343644Z 2021-12-10T12:56:17.881549Z 2021-12-10T12:56:17.684437Z 2021-12-10T12:56:17.434285Z 2021-12-10T12:56:16.917467Z 2021-12-10T12:56:16.853542Z 2021-12-10T12:56:16.463894Z 2021-12-10T12:56:16.058215Z 2021-12-10T12:56:15.812456Z 2021-12-10T12:56:15.205407Z 2021-12-10T12:56:15.458368Z 2021-12-10T12:56:14.924086Z 2021-12-10T12:56:14.562417Z 2021-12-10T12:56:14.012529Z 2021-12-10T12:56:14.015138Z 2021-12-10T12:56:13.207308Z 2021-12-10T12:56:12.949266Z 2021-12-10T12:56:12.558589Z 2021-12-10T12:56:12.119043Z 2021-12-10T12:56:11.838531Z 2021-12-10T12:56:10.889447Z 2021-12-10T12:56:11.405760Z 2021-12-10T12:56:10.648026Z 2021-12-10T12:56:10.337333Z 2021-12-10T12:56:10.179895Z 2021-12-10T12:56:09.713123Z 2021-12-10T12:56:09.575998Z 2021-12-10T12:56:09.372468Z 2021-12-10T12:56:08.885921Z 2021-12-10T12:56:08.908460Z 2021-12-10T12:56:08.643732Z 2021-12-10T12:56:08.702639Z 2021-12-10T12:56:08.518912Z 2021-12-10T12:56:08.392461Z 2021-12-10T12:56:01.393694Z 2021-12-10T12:56:01.037727Z 2021-12-10T12:55:59.912689Z 2021-12-10T12:55:59.855411Z 2021-12-10T12:56:00.157295Z 2021-12-10T12:55:59.618126Z 2021-12-10T12:55:59.607320Z 2021-12-10T12:55:59.008267Z 2021-12-10T12:55:57.868990Z 2021-12-10T12:55:57.518055Z 2021-12-10T12:55:57.237026Z 2021-12-10T12:55:56.655457Z 2021-12-10T12:55:51.753171Z 2021-12-10T12:55:51.399895Z 2021-12-10T12:55:51.076269Z 2021-12-10T12:55:50.167496Z 2021-12-10T12:55:50.090556Z 2021-12-10T13:00:58.406181Z 2021-12-10T13:00:58.054155Z 2021-12-10T13:00:57.692273Z 2021-12-10T13:00:56.962044Z 2021-12-10T13:00:56.361610Z 2021-12-10T13:00:55.956370Z 2021-12-10T13:00:55.697963Z 2021-12-10T13:00:55.254077Z 2021-12-10T13:00:54.710412Z 2021-12-10T13:00:54.383874Z 2021-12-10T13:00:53.915182Z 2021-12-10T13:00:53.409705Z 2021-12-10T13:00:52.442680Z 2021-12-10T13:00:52.333171Z 2021-12-10T13:00:51.853228Z 2021-12-10T13:00:51.541698Z 2021-12-10T13:00:51.348219Z 2021-12-10T13:00:51.168162Z 2021-12-10T13:00:50.589561Z 2021-12-10T13:00:50.093190Z 2021-12-10T13:00:49.817509Z 2021-12-10T13:00:49.481610Z 2021-12-10T13:00:49.170324Z 2021-12-10T13:00:48.662609Z 2021-12-10T13:00:48.683755Z 2021-12-10T13:00:47.507101Z 2021-12-10T13:00:46.551962Z 2021-12-10T13:00:46.793811Z 2021-12-10T13:00:45.912083Z 2021-12-10T13:00:45.669026Z 2021-12-10T13:00:45.528615Z 2021-12-10T13:00:45.216107Z 2021-12-10T13:00:44.519807Z 2021-12-10T13:00:44.187076Z 2021-12-10T13:00:43.743135Z 2021-12-10T13:00:43.404886Z 2021-12-10T13:00:43.112352Z 2021-12-10T13:00:43.110687Z 2021-12-10T13:00:42.228062Z 2021-12-10T13:00:42.287029Z 2021-12-10T13:00:41.344306Z 2021-12-10T13:00:40.987792Z 2021-12-10T13:00:40.689498Z 2021-12-10T13:00:40.487504Z 2021-12-10T13:00:40.056731Z 2021-12-10T13:00:39.574089Z 2021-12-10T13:00:39.344427Z 2021-12-10T13:00:39.218147Z 2021-12-10T13:00:38.847616Z 2021-12-10T13:00:38.448146Z 2021-12-10T13:00:38.229438Z 2021-12-10T13:00:38.112601Z 2021-12-10T13:00:36.965698Z 2021-12-10T13:00:36.519531Z 2021-12-10T13:00:36.248399Z 2021-12-10T13:00:35.512160Z 2021-12-10T13:00:34.997215Z 2021-12-10T13:00:34.614736Z 2021-12-10T13:00:34.133693Z 2021-12-10T13:00:33.936509Z 2021-12-10T13:00:33.552428Z 2021-12-10T13:00:33.248160Z 2021-12-10T13:00:32.955212Z 2021-12-10T13:00:32.796065Z 2021-12-10T13:00:32.480906Z 2021-12-10T13:00:32.314373Z 2021-12-10T13:00:31.908513Z 2021-12-10T13:00:31.582095Z 2021-12-10T13:00:31.407152Z 2021-12-10T13:00:31.228964Z 2021-12-10T13:00:31.202726Z 2021-12-10T13:00:30.957808Z 2021-12-10T13:00:30.708357Z 2021-12-10T13:00:30.644906Z 2021-12-10T13:00:23.179255Z 2021-12-10T13:00:23.000620Z 2021-12-10T13:00:22.690678Z 2021-12-10T13:00:22.668510Z 2021-12-10T13:00:22.335351Z 2021-12-10T13:00:21.835241Z 2021-12-10T13:00:22.368138Z 2021-12-10T13:00:21.511248Z 2021-12-10T13:00:21.530693Z 2021-12-10T13:00:21.283690Z 2021-12-10T13:00:20.963225Z 2021-12-10T13:00:20.821276Z 2021-12-10T13:00:20.575892Z 2021-12-10T13:00:20.438179Z 2021-12-10T13:00:19.865605Z 2021-12-10T13:00:19.933666Z 2021-12-10T13:00:19.603249Z 2021-12-10T13:00:19.785206Z 2021-12-10T13:00:19.293995Z 2021-12-10T13:00:19.507090Z 2021-12-10T13:00:19.000646Z 2021-12-10T13:00:19.037899Z 2021-12-10T13:00:18.715129Z 2021-12-10T13:00:11.827440Z 2021-12-10T13:00:09.866434Z 2021-12-10T13:00:09.782048Z", + "started_diff": 50.0904778, + "successful_job_ids": " [99, 98, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 75, 74, 73, 72, 71, 70, 68, 67, 66, 65, 63, 62, 61, 60, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 151, 149, 148, 147, 145, 144, 143, 142, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 106, 105, 104, 103, 101, 100, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472]" + }, + "started": "2021-12-10T13:27:04.931804+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-api.json new file mode 100644 index 00000000..2c7eb24a --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-10T19:04:35.339880+00:00", + "golden": "true", + "id": "run-2021-12-10T14_50_06_287_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2409.7333333333327, + "max": 23600.800000000003, + "mean": 5340.094157716393, + "median": 3366.0, + "min": 1320.5333333333333, + "non_zero_mean": 5340.094157716393, + "non_zero_median": 3366.0, + "percentile25": 2792.2, + "percentile75": 5201.9333333333325, + "percentile90": 12217.133333333333, + "percentile99": 23313.466666666664, + "percentile999": 23546.267013132463, + "range": 22280.26666666667, + "samples": 261, + "stdev": 4747.624159934627, + "sum": 1393764.5751639782 + } + }, + "cpu": { + "system": { + "iqr": 0.18400000000001834, + "max": 1.7320000000000049, + "mean": 0.2681225674148203, + "median": 0.11266666666666272, + "min": 0.02866666666665954, + "non_zero_mean": 0.2681225674148203, + "non_zero_median": 0.11266666666666272, + "percentile25": 0.06066666666665696, + "percentile75": 0.2446666666666753, + "percentile90": 0.8373333333333374, + "percentile99": 1.6324, + "percentile999": 1.7151866666666735, + "range": 1.7033333333333454, + "samples": 261, + "stdev": 0.3788817642058565, + "sum": 69.97999009526812 + }, + "user": { + "iqr": 0.5513333333333472, + "max": 6.403333333333376, + "mean": 1.375011536592093, + "median": 0.7633333333333667, + "min": 0.41000000000003634, + "non_zero_mean": 1.375011536592093, + "non_zero_median": 0.7633333333333667, + "percentile25": 0.5999999999999697, + "percentile75": 1.1513333333333169, + "percentile90": 3.7993333333333643, + "percentile99": 6.161199999999992, + "percentile999": 6.361040000000043, + "range": 5.993333333333339, + "samples": 261, + "stdev": 1.466154737251898, + "sum": 358.8780110505362 + } + }, + "entropy_available_bits": { + "iqr": 1.8666666666666667, + "max": 215.46666666666667, + "mean": 20.42911877394636, + "median": 1.7333333333333334, + "min": 0.2, + "non_zero_mean": 20.42911877394636, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.7333333333333334, + "percentile75": 2.6, + "percentile90": 6.866666666666667, + "percentile99": 213.82666666666668, + "percentile999": 215.08533333333335, + "range": 215.26666666666668, + "samples": 261, + "stdev": 57.068880289995064, + "sum": 5331.999999999999 + }, + "memory": { + "used": { + "iqr": 324866048.0, + "max": 14814412800.0, + "mean": 6428252152.153256, + "median": 5970206720.0, + "min": 5718298624.0, + "non_zero_mean": 6428252152.153256, + "non_zero_median": 5970206720.0, + "percentile25": 5861515264.0, + "percentile75": 6186381312.0, + "percentile90": 7144955904.0, + "percentile99": 12904246476.79997, + "percentile999": 14721091420.160004, + "range": 9096114176.0, + "samples": 261, + "stdev": 1447020422.0725493, + "sum": 1677773811712.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 261, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 85.13333333333334, + "max": 1356.4684709908229, + "mean": 116.73507659039117, + "median": 27.4, + "min": 0.0, + "non_zero_mean": 119.01505855504725, + "non_zero_median": 31.033333333333335, + "percentile25": 1.5333333333333332, + "percentile75": 86.66666666666667, + "percentile90": 352.4, + "percentile99": 1206.4399999999998, + "percentile999": 1344.5080018665426, + "range": 1356.4684709908229, + "samples": 261, + "stdev": 244.46342418421352, + "sum": 30467.8549900921 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 978.1999999999998, + "max": 3038.2, + "mean": 1621.530779054917, + "median": 1470.6666666666667, + "min": 470.53333333333336, + "non_zero_mean": 1621.530779054917, + "non_zero_median": 1470.6666666666667, + "percentile25": 1137.4666666666667, + "percentile75": 2115.6666666666665, + "percentile90": 2452.8, + "percentile99": 2979.093333333332, + "percentile999": 3035.964, + "range": 2567.6666666666665, + "samples": 261, + "stdev": 600.0334144202312, + "sum": 423219.5333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.04466666666666584, + "max": 0.2593333333333343, + "mean": 0.08603575989782886, + "median": 0.07333333333333296, + "min": 0.013333333333332576, + "non_zero_mean": 0.08603575989782886, + "non_zero_median": 0.07333333333333296, + "percentile25": 0.05466666666666621, + "percentile75": 0.09933333333333205, + "percentile90": 0.15999999999999848, + "percentile99": 0.2259999999999984, + "percentile999": 0.25274666666666795, + "range": 0.24600000000000172, + "samples": 261, + "stdev": 0.045724878657872586, + "sum": 22.455333333333332 + }, + "user": { + "iqr": 0.12133333333333668, + "max": 0.928000000000001, + "mean": 0.20492720306513412, + "median": 0.1946666666666696, + "min": 0.017999999999995, + "non_zero_mean": 0.20492720306513412, + "non_zero_median": 0.1946666666666696, + "percentile25": 0.1400000000000015, + "percentile75": 0.2613333333333382, + "percentile90": 0.32666666666666516, + "percentile99": 0.5601333333333313, + "percentile999": 0.846013333333336, + "range": 0.910000000000006, + "samples": 261, + "stdev": 0.10796507933326249, + "sum": 53.48599999999997 + } + }, + "entropy_available_bits": { + "iqr": 1.5333333333333334, + "max": 211.26666666666668, + "mean": 11.333077905491699, + "median": 0.6666666666666666, + "min": 0.0, + "non_zero_mean": 20.541203703703705, + "non_zero_median": 1.4, + "percentile25": 0.0, + "percentile75": 1.5333333333333334, + "percentile90": 3.2, + "percentile99": 210.96, + "percentile999": 211.232, + "range": 211.26666666666668, + "samples": 261, + "stdev": 45.77310765667861, + "sum": 2957.933333333334 + }, + "memory": { + "used": { + "iqr": 21483520.0, + "max": 1101045760.0, + "mean": 426807123.37164754, + "median": 385236992.0, + "min": 365182976.0, + "non_zero_mean": 426807123.37164754, + "non_zero_median": 385236992.0, + "percentile25": 377458688.0, + "percentile75": 398942208.0, + "percentile90": 503472128.0, + "percentile99": 989420748.7999976, + "percentile999": 1089757184.0000005, + "range": 735862784.0, + "samples": 261, + "stdev": 125538102.25370017, + "sum": 111396659200.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6007.466666666666, + "max": 59735576.46666667, + "mean": 1576406.0222222223, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 4241669.812371134, + "non_zero_median": 10922.666666666666, + "percentile25": 0.0, + "percentile75": 6007.466666666666, + "percentile90": 41506.13333333333, + "percentile99": 57966213.693333335, + "percentile999": 59650237.672000006, + "range": 59735576.46666667, + "samples": 261, + "stdev": 9445701.986416204, + "sum": 411441971.7999999 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 421.983213, + "mean": 5.536130478927203, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 46.61064693548387, + "non_zero_median": 0.166701, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.008494, + "percentile99": 193.99306239999933, + "percentile999": 402.4841224800007, + "range": 421.983213, + "samples": 261, + "stdev": 39.166620537833396, + "sum": 1444.9300549999998 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.252417, + "max": 406.992942, + "mean": 21.837802555555555, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 50.88987916964286, + "non_zero_median": 0.638936, + "percentile25": 0.0, + "percentile75": 0.252417, + "percentile90": 47.33505, + "percentile99": 367.98756439999966, + "percentile999": 403.09125616000017, + "range": 406.992942, + "samples": 261, + "stdev": 71.27369474316659, + "sum": 5699.666466999998 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 6.466666666666667, + "mean": 0.16372924648786719, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.287179487179487, + "non_zero_median": 2.8666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.85333333333333, + "percentile999": 5.9813333333333505, + "range": 6.466666666666667, + "samples": 261, + "stdev": 0.7688235356534596, + "sum": 42.73333333333333 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9999.733333333334, + "mean": 442.8309067688378, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8890.68205128205, + "non_zero_median": 9083.6, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9975.88, + "percentile999": 9997.844000000001, + "range": 9999.733333333334, + "samples": 261, + "stdev": 1957.291059798617, + "sum": 115578.86666666667 + }, + "pg_stat_database_blks_hit": { + "iqr": 22210.26666666667, + "max": 211864.73333333334, + "mean": 26264.482247765005, + "median": 26263.466666666667, + "min": 2599.9333333333334, + "non_zero_mean": 26264.482247765005, + "non_zero_median": 26263.466666666667, + "percentile25": 11881.0, + "percentile75": 34091.26666666667, + "percentile90": 45817.53333333333, + "percentile99": 69568.97333333331, + "percentile999": 199530.12533333377, + "range": 209264.80000000002, + "samples": 261, + "stdev": 20787.996367724463, + "sum": 6855029.8666666625 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5333333333333333, + "max": 213.13333333333333, + "mean": 1.4102171136653896, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.8589225589225589, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.8666666666666667, + "percentile99": 4.413333333333329, + "percentile999": 159.74666666666855, + "range": 213.13333333333333, + "samples": 261, + "stdev": 13.193296581835416, + "sum": 368.0666666666663 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 261, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5333333333333333, + "max": 213.13333333333333, + "mean": 1.4102171136653896, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.8589225589225589, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.8666666666666667, + "percentile99": 4.413333333333329, + "percentile999": 159.74666666666855, + "range": 213.13333333333333, + "samples": 261, + "stdev": 13.193296581835416, + "sum": 368.0666666666663 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.6666666666666665, + "max": 4006.733333333333, + "mean": 41.59131545338442, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 102.40880503144653, + "non_zero_median": 14.466666666666665, + "percentile25": 0.0, + "percentile75": 2.6666666666666665, + "percentile90": 59.53333333333333, + "percentile99": 90.66666666666654, + "percentile999": 4005.4333333333334, + "range": 4006.733333333333, + "samples": 261, + "stdev": 349.61341711748145, + "sum": 10855.333333333332 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11406.333333333332, + "max": 980151.3333333334, + "mean": 17415.253895274585, + "median": 13863.8, + "min": 1288.2666666666667, + "non_zero_mean": 17415.253895274585, + "non_zero_median": 13863.8, + "percentile25": 6290.933333333333, + "percentile75": 17697.266666666666, + "percentile90": 26085.4, + "percentile99": 38106.79999999994, + "percentile999": 736169.8120000085, + "range": 978863.0666666667, + "samples": 261, + "stdev": 60507.57458885021, + "sum": 4545381.266666665 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.933333333333334, + "max": 8005.333333333333, + "mean": 46.56704980842912, + "median": 2.533333333333333, + "min": 0.0, + "non_zero_mean": 56.00921658986175, + "non_zero_median": 3.8666666666666667, + "percentile25": 1.1333333333333333, + "percentile75": 10.066666666666666, + "percentile90": 66.46666666666667, + "percentile99": 143.06666666666666, + "percentile999": 5965.841333333405, + "range": 8005.333333333333, + "samples": 261, + "stdev": 495.65804864201823, + "sum": 12153.99999999999 + }, + "pg_stat_database_tup_returned": { + "iqr": 37298.53333333334, + "max": 1016148.6, + "mean": 48472.44010217114, + "median": 36829.46666666667, + "min": 3383.866666666667, + "non_zero_mean": 48472.44010217114, + "non_zero_median": 36829.46666666667, + "percentile25": 19434.8, + "percentile75": 56733.333333333336, + "percentile90": 96168.46666666666, + "percentile99": 193442.55999999933, + "percentile999": 893713.9413333376, + "range": 1012764.7333333333, + "samples": 261, + "stdev": 75954.73122320655, + "sum": 12651306.866666665 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.6, + "max": 63.13333333333333, + "mean": 10.45389527458493, + "median": 7.333333333333333, + "min": 0.13333333333333333, + "non_zero_mean": 10.45389527458493, + "non_zero_median": 7.333333333333333, + "percentile25": 3.466666666666667, + "percentile75": 16.066666666666666, + "percentile90": 23.8, + "percentile99": 45.06666666666664, + "percentile999": 61.105333333333405, + "range": 63.0, + "samples": 261, + "stdev": 10.02562394115714, + "sum": 2728.4666666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 125.13333333333333, + "max": 710.9333333333333, + "mean": 124.39003831417625, + "median": 90.2, + "min": 4.066666666666666, + "non_zero_mean": 124.39003831417625, + "non_zero_median": 90.2, + "percentile25": 40.333333333333336, + "percentile75": 165.46666666666667, + "percentile90": 280.26666666666665, + "percentile99": 591.9866666666649, + "percentile999": 694.6226666666672, + "range": 706.8666666666666, + "samples": 261, + "stdev": 125.16664127640931, + "sum": 32465.799999999985 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.1269476372924649, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.1269476372924649, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 261, + "stdev": 0.1452734153116778, + "sum": 33.13333333333333 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 261, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 24.53333333333333, + "max": 150.39999999999998, + "mean": 39.506513409961684, + "median": 32.8, + "min": 1.6, + "non_zero_mean": 39.506513409961684, + "non_zero_median": 32.8, + "percentile25": 23.46666666666667, + "percentile75": 48.0, + "percentile90": 74.13333333333334, + "percentile99": 122.49333333333314, + "percentile999": 146.7080000000001, + "range": 148.79999999999998, + "samples": 261, + "stdev": 23.979100857311977, + "sum": 10311.199999999997 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-10T17:59:17.754278+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-10T14_50_06_287_0000", + "ended": "2021-12-10T19:04:29.916480+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.016416339749866893, + "iterations": 1, + "max": 0.5644362909997653, + "mean": 0.5474970785999782, + "median": 0.5484608899996601, + "min": 0.5338978629997655, + "rounds": 15, + "stddev": 0.009577429831521173 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.018786411750397747, + "iterations": 1, + "max": 0.5552536969998982, + "mean": 0.5385950041998513, + "median": 0.5404344739999942, + "min": 0.48722699099926103, + "rounds": 15, + "stddev": 0.017024038493878207 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.007206614000097034, + "iterations": 1, + "max": 0.4033371839996107, + "mean": 0.3897651020000922, + "median": 0.3882811405001121, + "min": 0.3830876680003712, + "rounds": 10, + "stddev": 0.0071844260248166494 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.01043055500031187, + "iterations": 1, + "max": 0.3601893270006258, + "mean": 0.26281857030003264, + "median": 0.259260452000035, + "min": 0.24813759900007426, + "rounds": 100, + "stddev": 0.015354684575926666 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.013463559499086841, + "iterations": 1, + "max": 0.5271661199994924, + "mean": 0.46931180921988924, + "median": 0.4680003469993608, + "min": 0.44958901299924037, + "rounds": 100, + "stddev": 0.012017902865684485 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.07103156100038177, + "iterations": 1, + "max": 1.487980157000493, + "mean": 1.3839331383001992, + "median": 1.3756148585002848, + "min": 1.2755377149997003, + "rounds": 10, + "stddev": 0.06320246687558147 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0013697149988729507, + "iterations": 1, + "max": 0.08393008500024735, + "mean": 0.07459326864292441, + "median": 0.07372465150001517, + "min": 0.07236636099969473, + "rounds": 14, + "stddev": 0.0028847964982660792 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0017469222507315862, + "iterations": 1, + "max": 0.07542508599999564, + "mean": 0.0720655755999663, + "median": 0.07243991899940738, + "min": 0.06795555600001535, + "rounds": 15, + "stddev": 0.0019142162049765612 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.0014602639994336641, + "iterations": 1, + "max": 0.10578405499927612, + "mean": 0.09690136679973875, + "median": 0.09599811099997169, + "min": 0.09419387900015863, + "rounds": 10, + "stddev": 0.0033011936282834192 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.003885163999257202, + "iterations": 1, + "max": 0.11671368999941478, + "mean": 0.10934106320000865, + "median": 0.10911705050011733, + "min": 0.10442993400010891, + "rounds": 10, + "stddev": 0.003588756121551009 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.008726325999305118, + "iterations": 1, + "max": 0.3989387830006308, + "mean": 0.28648855769997683, + "median": 0.27625574600006075, + "min": 0.2610970690002432, + "rounds": 10, + "stddev": 0.04004063852633962 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.003590498999983538, + "iterations": 1, + "max": 0.13265146300000197, + "mean": 0.11366600290029964, + "median": 0.11049036700069337, + "min": 0.10826176200043847, + "rounds": 10, + "stddev": 0.0078042121953617325 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0023521410003013443, + "iterations": 1, + "max": 0.07382434100054525, + "mean": 0.07109810785738903, + "median": 0.07179115300050398, + "min": 0.06688531000054354, + "rounds": 14, + "stddev": 0.0020276996417226286 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.01544617899980949, + "iterations": 1, + "max": 0.6549761400001444, + "mean": 0.6282132886998625, + "median": 0.6245625529995777, + "min": 0.6159680119999393, + "rounds": 10, + "stddev": 0.012788196131164824 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0069399247499859484, + "iterations": 1, + "max": 0.10077274100058276, + "mean": 0.09473403145470498, + "median": 0.09305923499960045, + "min": 0.08991758299998764, + "rounds": 11, + "stddev": 0.003995291832722542 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.0027520149988049525, + "iterations": 1, + "max": 0.11395882399938273, + "mean": 0.10686341479995462, + "median": 0.10627982350024467, + "min": 0.10185798399925261, + "rounds": 10, + "stddev": 0.0036660640473369513 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.007182073999501881, + "iterations": 1, + "max": 0.20182296300026792, + "mean": 0.1815484689999721, + "median": 0.17864356399968528, + "min": 0.16774037600043812, + "rounds": 10, + "stddev": 0.00973184581299039 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.00498653100021329, + "iterations": 1, + "max": 0.10067244300080347, + "mean": 0.07721974928576206, + "median": 0.0750238154996623, + "min": 0.07116701799986913, + "rounds": 14, + "stddev": 0.007221065254369893 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.0024517444994671678, + "iterations": 1, + "max": 0.08077617100025236, + "mean": 0.07720607400005974, + "median": 0.07693184000072506, + "min": 0.07501459600007365, + "rounds": 12, + "stddev": 0.0018115645440852778 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.017132535000200733, + "iterations": 1, + "max": 0.39374217899967334, + "mean": 0.3798541030002525, + "median": 0.38205587000038577, + "min": 0.3657386810000389, + "rounds": 10, + "stddev": 0.009736009883577552 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.017330270999991626, + "iterations": 1, + "max": 0.5049037149992728, + "mean": 0.3965929281001081, + "median": 0.37772475800011307, + "min": 0.3710545500007356, + "rounds": 10, + "stddev": 0.043074340451167724 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.0031657599993195618, + "iterations": 1, + "max": 0.10230559599949629, + "mean": 0.09529885769998145, + "median": 0.09414599100000487, + "min": 0.0924820969994471, + "rounds": 10, + "stddev": 0.0030161107356359964 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.0022303030004877655, + "iterations": 1, + "max": 0.09190750800007663, + "mean": 0.08586242800023076, + "median": 0.08560032150035113, + "min": 0.0822443890001523, + "rounds": 12, + "stddev": 0.002367347863244283 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.008960982000644435, + "iterations": 1, + "max": 0.24654779499996948, + "mean": 0.1812585685999693, + "median": 0.17400127950031674, + "min": 0.1664623169999686, + "rounds": 10, + "stddev": 0.023554794782085553 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0016339915000571636, + "iterations": 1, + "max": 0.07677370799956407, + "mean": 0.07259207206661814, + "median": 0.07203665900033229, + "min": 0.06999524799994106, + "rounds": 15, + "stddev": 0.0016613930705470556 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.010420929998872452, + "iterations": 1, + "max": 0.214088895999339, + "mean": 0.20796938469984524, + "median": 0.20829867500015098, + "min": 0.20047276899913413, + "rounds": 10, + "stddev": 0.005109316126285341 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0021407444999113068, + "iterations": 1, + "max": 0.09132664700064197, + "mean": 0.08202193069246277, + "median": 0.08071669899982226, + "min": 0.07921864299987647, + "rounds": 13, + "stddev": 0.0038397681307091036 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.01982051899904036, + "iterations": 1, + "max": 0.3867347310006153, + "mean": 0.36852583710006004, + "median": 0.36834046300009504, + "min": 0.3511565299995709, + "rounds": 10, + "stddev": 0.0125820469707085 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.005140432999723998, + "iterations": 1, + "max": 0.2359702820003804, + "mean": 0.23114414680003392, + "median": 0.2313949305003007, + "min": 0.22665846799918654, + "rounds": 10, + "stddev": 0.003336066502819847 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.004198086000542389, + "iterations": 1, + "max": 0.1667892169998595, + "mean": 0.1624436569998579, + "median": 0.16319601299983333, + "min": 0.15803775599943037, + "rounds": 10, + "stddev": 0.002842883106776127 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.0027007910002794233, + "iterations": 1, + "max": 0.23882776299979014, + "mean": 0.13501452900000005, + "median": 0.10938549349975801, + "min": 0.10806318699997064, + "rounds": 10, + "stddev": 0.05402677263414009 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.002065852749865371, + "iterations": 1, + "max": 0.07477331500012951, + "mean": 0.07157855893347005, + "median": 0.07126915000026202, + "min": 0.06840760199975193, + "rounds": 15, + "stddev": 0.0017052995897228567 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.0049137659998450545, + "iterations": 1, + "max": 0.10227017899978819, + "mean": 0.09660257472726451, + "median": 0.09634040999935678, + "min": 0.0921575259999372, + "rounds": 11, + "stddev": 0.003116324726496987 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0018818949993146816, + "iterations": 1, + "max": 0.07703979999951116, + "mean": 0.07394753735700631, + "median": 0.07442298599971764, + "min": 0.07044223899993085, + "rounds": 14, + "stddev": 0.0019142037079120226 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.005663302999892039, + "iterations": 1, + "max": 0.13612384800035215, + "mean": 0.125541841499944, + "median": 0.1255577759998232, + "min": 0.1181860870001401, + "rounds": 10, + "stddev": 0.005174359564365621 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0037924730004306184, + "iterations": 1, + "max": 0.07959798499996396, + "mean": 0.07535527157155489, + "median": 0.07488994650020686, + "min": 0.07251105800060031, + "rounds": 14, + "stddev": 0.002318050686220048 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.09750584799985518, + "iterations": 1, + "max": 0.7543416090002211, + "mean": 0.6615016326999467, + "median": 0.6439013665003586, + "min": 0.6018658180000784, + "rounds": 10, + "stddev": 0.05508521482240593 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.01247379800042836, + "iterations": 1, + "max": 0.7494458890005262, + "mean": 0.6453251281001939, + "median": 0.6372623995002868, + "min": 0.605949580000015, + "rounds": 10, + "stddev": 0.03959238815348439 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.013862131000678346, + "iterations": 1, + "max": 0.2894929449994379, + "mean": 0.27432940779999626, + "median": 0.2749874690002798, + "min": 0.26518901100007497, + "rounds": 10, + "stddev": 0.007921525076792045 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0011281257491191354, + "iterations": 1, + "max": 0.08007179099968198, + "mean": 0.07281886946139611, + "median": 0.07225550099974498, + "min": 0.07094566899922938, + "rounds": 13, + "stddev": 0.0023868271258863927 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0021993899999870337, + "iterations": 1, + "max": 0.07480356199994276, + "mean": 0.07189230114279001, + "median": 0.07230529649996242, + "min": 0.06972720799967647, + "rounds": 14, + "stddev": 0.0014739336624229964 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.001971214000150212, + "iterations": 1, + "max": 0.08099131400012993, + "mean": 0.07497599971429736, + "median": 0.07446230099958484, + "min": 0.06999898300000496, + "rounds": 14, + "stddev": 0.003014750311384865 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.003626245999839739, + "iterations": 1, + "max": 0.07968054299999494, + "mean": 0.07490735235712366, + "median": 0.07400847049984804, + "min": 0.07188438800039876, + "rounds": 14, + "stddev": 0.0023094814222890713 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.01336898600038694, + "iterations": 1, + "max": 0.3288558180001928, + "mean": 0.29336710520001363, + "median": 0.293471199999658, + "min": 0.26249106700015545, + "rounds": 10, + "stddev": 0.017464221454749025 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.005771332999756851, + "iterations": 1, + "max": 0.250343681999766, + "mean": 0.22997982950000734, + "median": 0.22817876899989642, + "min": 0.21765476500058867, + "rounds": 10, + "stddev": 0.009227314825577405 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.033621321999817155, + "iterations": 1, + "max": 1.0226599099996747, + "mean": 0.9180601777999072, + "median": 0.9422117099998104, + "min": 0.6355793149996316, + "rounds": 10, + "stddev": 0.10357880209030246 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.09051870200073608, + "iterations": 1, + "max": 8.774813874999381, + "mean": 8.715224063799814, + "median": 8.738277270000253, + "min": 8.630494748999809, + "rounds": 10, + "stddev": 0.05208998797853252 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.018460895999851346, + "iterations": 1, + "max": 1.0670453109996743, + "mean": 1.0315917081999033, + "median": 1.0272680039997795, + "min": 1.0132554159999927, + "rounds": 10, + "stddev": 0.01544592538120988 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.20347644800040143, + "iterations": 1, + "max": 9.216049446999932, + "mean": 9.00990318029999, + "median": 8.98546517199975, + "min": 8.831812458000059, + "rounds": 10, + "stddev": 0.13444374297164224 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.25681926000015665, + "iterations": 1, + "max": 1.5155053680000492, + "mean": 1.1869431627000266, + "median": 1.1162226955002552, + "min": 1.0181099419996826, + "rounds": 10, + "stddev": 0.18549546303627062 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.015087203000803129, + "iterations": 1, + "max": 0.4122207899999921, + "mean": 0.39731694459997013, + "median": 0.39809692900007576, + "min": 0.3790854019998733, + "rounds": 10, + "stddev": 0.009761076953128974 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.09953570200013928, + "iterations": 1, + "max": 11.739602956000454, + "mean": 11.352989395400437, + "median": 11.327738068999679, + "min": 11.125209688001632, + "rounds": 10, + "stddev": 0.16072058443830878 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.029366068999479467, + "iterations": 1, + "max": 0.44201842499933264, + "mean": 0.4035289146999276, + "median": 0.4072473904998333, + "min": 0.3706970420007565, + "rounds": 10, + "stddev": 0.022370352505504527 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.17866648600011104, + "iterations": 1, + "max": 1.0341372190005131, + "mean": 0.5798024705999524, + "median": 0.5246784895002747, + "min": 0.41806908099988505, + "rounds": 10, + "stddev": 0.18909343738283127 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.0953433300001052, + "iterations": 1, + "max": 11.99369293000018, + "mean": 11.653224046899776, + "median": 11.640976884999873, + "min": 11.482298763001381, + "rounds": 10, + "stddev": 0.1396927645904052 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.06620445199951064, + "iterations": 1, + "max": 7.906539188000352, + "mean": 7.755638743000054, + "median": 7.749605629499911, + "min": 7.684751134999715, + "rounds": 10, + "stddev": 0.06404745902601297 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 269.22615172100086, + "mean": 269.22615172100086, + "median": 269.22615172100086, + "min": 269.22615172100086, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 397.7205420240007, + "mean": 397.7205420240007, + "median": 397.7205420240007, + "min": 397.7205420240007, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.009310536000157299, + "iterations": 1, + "max": 0.26115106899942475, + "mean": 0.25538064069978644, + "median": 0.2558263324995096, + "min": 0.24673821200030943, + "rounds": 10, + "stddev": 0.005476608115441833 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.008073963001152151, + "iterations": 1, + "max": 0.7398660429998927, + "mean": 0.6441149983000287, + "median": 0.6361250165000456, + "min": 0.6183146189996478, + "rounds": 10, + "stddev": 0.034388971861649534 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.2581250250004814, + "iterations": 1, + "max": 9.709703141999853, + "mean": 9.52509529850031, + "median": 9.581133072000739, + "min": 9.300857280000855, + "rounds": 10, + "stddev": 0.14552449206617665 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.40200788299898704, + "iterations": 1, + "max": 4.013791962000141, + "mean": 3.073410420300024, + "median": 3.0529077959999995, + "min": 2.2532255639998766, + "rounds": 10, + "stddev": 0.4782281291166255 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.7142607970008612, + "iterations": 1, + "max": 2.4751279760002944, + "mean": 1.95026444560026, + "median": 1.9475270710004224, + "min": 1.4896967650001898, + "rounds": 10, + "stddev": 0.3495748877912921 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.5387410620005539, + "iterations": 1, + "max": 25.90529747799883, + "mean": 24.639594217399825, + "median": 24.473431806000008, + "min": 23.8619956750008, + "rounds": 10, + "stddev": 0.6470219192197554 + } + }, + "started": "2021-12-10T17:48:33.003959+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-chatty.json new file mode 100644 index 00000000..a9ab9f53 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-10T17:02:13.236089+00:00", + "golden": "true", + "id": "run-2021-12-10T14_50_06_287_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 14872.133333333328, + "max": 43975.8, + "mean": 31311.641025641027, + "median": 38195.26666666666, + "min": 2003.9333333333334, + "non_zero_mean": 31311.641025641027, + "non_zero_median": 38195.26666666666, + "percentile25": 24495.266666666666, + "percentile75": 39367.399999999994, + "percentile90": 40272.74666666667, + "percentile99": 42386.46666666667, + "percentile999": 43816.86666666669, + "range": 41971.86666666667, + "samples": 65, + "stdev": 12753.59381833265, + "sum": 2035256.6666666667 + } + }, + "cpu": { + "system": { + "iqr": 0.21733333333333582, + "max": 1.0620000000000025, + "mean": 0.8341333333333333, + "median": 1.001333333333334, + "min": 0.04066666666666757, + "non_zero_mean": 0.8341333333333333, + "non_zero_median": 1.001333333333334, + "percentile25": 0.7993333333333319, + "percentile75": 1.0166666666666677, + "percentile90": 1.0324000000000009, + "percentile99": 1.0590133333333343, + "percentile999": 1.0617013333333356, + "range": 1.021333333333335, + "samples": 65, + "stdev": 0.30837968570506635, + "sum": 54.21866666666668 + }, + "user": { + "iqr": 2.262666666666658, + "max": 6.589333333333329, + "mean": 5.0715076923076925, + "median": 6.33400000000001, + "min": 0.252000000000002, + "non_zero_mean": 5.0715076923076925, + "non_zero_median": 6.33400000000001, + "percentile25": 4.1300000000000034, + "percentile75": 6.392666666666662, + "percentile90": 6.425600000000015, + "percentile99": 6.510399999999998, + "percentile999": 6.581439999999996, + "range": 6.337333333333326, + "samples": 65, + "stdev": 2.0589411561289714, + "sum": 329.64799999999985 + } + }, + "entropy_available_bits": { + "iqr": 8.066666666666666, + "max": 215.46666666666664, + "mean": 22.595897435897434, + "median": 7.2, + "min": 0.0, + "non_zero_mean": 31.24964539007092, + "non_zero_median": 7.7333333333333325, + "percentile25": 0.0, + "percentile75": 8.066666666666666, + "percentile90": 15.586666666666668, + "percentile99": 212.69333333333333, + "percentile999": 215.18933333333334, + "range": 215.46666666666664, + "samples": 65, + "stdev": 56.192407401468834, + "sum": 1468.7333333333331 + }, + "memory": { + "used": { + "iqr": 1367511040.0, + "max": 7480553472.0, + "mean": 6394151668.184615, + "median": 6838747136.0, + "min": 3497160704.0, + "non_zero_mean": 6394151668.184615, + "non_zero_median": 6838747136.0, + "percentile25": 5736611840.0, + "percentile75": 7104122880.0, + "percentile90": 7249942118.4, + "percentile99": 7450333511.68, + "percentile999": 7477531475.968, + "range": 3983392768.0, + "samples": 65, + "stdev": 952443325.9785526, + "sum": 415619858432.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.006153846153846154, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2, + "non_zero_median": 0.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.18133333333333326, + "percentile999": 0.25813333333333427, + "range": 0.26666666666666666, + "samples": 65, + "stdev": 0.03674816118430078, + "sum": 0.4 + } + }, + "writes_completed": { + "total": { + "iqr": 2520.3333333333335, + "max": 4973.0, + "mean": 3510.2297435897435, + "median": 4512.533333333333, + "min": 14.600000000000001, + "non_zero_mean": 3510.2297435897435, + "non_zero_median": 4512.533333333333, + "percentile25": 2134.4, + "percentile75": 4654.733333333334, + "percentile90": 4773.5199999999995, + "percentile99": 4905.757333333333, + "percentile999": 4966.275733333334, + "range": 4958.4, + "samples": 65, + "stdev": 1623.599664797961, + "sum": 228164.93333333332 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 254.5333333333333, + "max": 2163.133333333333, + "mean": 1627.8584615384616, + "median": 1703.7333333333333, + "min": 616.4, + "non_zero_mean": 1627.8584615384616, + "non_zero_median": 1703.7333333333333, + "percentile25": 1563.6666666666667, + "percentile75": 1818.2, + "percentile90": 1928.1466666666668, + "percentile99": 2146.365333333333, + "percentile999": 2161.4565333333335, + "range": 1546.7333333333331, + "samples": 65, + "stdev": 335.9451841255943, + "sum": 105810.8 + } + }, + "cpu": { + "system": { + "iqr": 0.04400000000000026, + "max": 0.244, + "mean": 0.14312820512820512, + "median": 0.15599999999999975, + "min": 0.023333333333333546, + "non_zero_mean": 0.14312820512820512, + "non_zero_median": 0.15599999999999975, + "percentile25": 0.12333333333333296, + "percentile75": 0.16733333333333322, + "percentile90": 0.18346666666666675, + "percentile99": 0.2354666666666667, + "percentile999": 0.24314666666666676, + "range": 0.22066666666666646, + "samples": 65, + "stdev": 0.04590886550138147, + "sum": 9.303333333333333 + }, + "user": { + "iqr": 0.04333333333333325, + "max": 0.5746666666666661, + "mean": 0.22247179487179486, + "median": 0.1780000000000001, + "min": 0.03399999999999939, + "non_zero_mean": 0.22247179487179486, + "non_zero_median": 0.1780000000000001, + "percentile25": 0.16600000000000012, + "percentile75": 0.20933333333333337, + "percentile90": 0.5019999999999999, + "percentile99": 0.56528, + "percentile999": 0.5737279999999996, + "range": 0.5406666666666667, + "samples": 65, + "stdev": 0.1407761581710543, + "sum": 14.460666666666674 + } + }, + "entropy_available_bits": { + "iqr": 1.5333333333333332, + "max": 186.0, + "mean": 9.61025641025641, + "median": 2.0, + "min": 0.3333333333333333, + "non_zero_mean": 9.61025641025641, + "non_zero_median": 2.0, + "percentile25": 1.4, + "percentile75": 2.933333333333333, + "percentile90": 4.4, + "percentile99": 174.52266666666665, + "percentile999": 184.8522666666668, + "range": 185.66666666666666, + "samples": 65, + "stdev": 34.157017399164666, + "sum": 624.6666666666665 + }, + "memory": { + "used": { + "iqr": 22851584.0, + "max": 530788352.0, + "mean": 451125500.06153846, + "median": 455315456.0, + "min": 358555648.0, + "non_zero_mean": 451125500.06153846, + "non_zero_median": 455315456.0, + "percentile25": 443191296.0, + "percentile75": 466042880.0, + "percentile90": 519109836.8, + "percentile99": 528405463.04, + "percentile999": 530550063.10400003, + "range": 172232704.0, + "samples": 65, + "stdev": 41694185.90564761, + "sum": 29323157504.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1800055.4666666666, + "max": 3413879.466666667, + "mean": 791649.6738461539, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1905823.288888889, + "non_zero_median": 2069845.3333333333, + "percentile25": 0.0, + "percentile75": 1800055.4666666666, + "percentile90": 2875282.773333335, + "percentile99": 3406888.96, + "percentile999": 3413180.416, + "range": 3413879.466666667, + "samples": 65, + "stdev": 1182372.738766925, + "sum": 51457228.8 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.006146, + "max": 5.400259, + "mean": 0.8848480307692308, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.195284555555556, + "non_zero_median": 5.1082184999999996, + "percentile25": 0.0, + "percentile75": 0.006146, + "percentile90": 5.1693286, + "percentile99": 5.34721836, + "percentile999": 5.394954936, + "range": 5.400259, + "samples": 65, + "stdev": 1.9730205335825195, + "sum": 57.515122000000005 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 1.504426, + "mean": 0.0366194, + "median": 0.0, + "min": -0.000379, + "non_zero_mean": 0.19835508333333335, + "non_zero_median": 0.031016000000000002, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.024310000000000016, + "percentile99": 0.6928042399999993, + "percentile999": 1.423263824000009, + "range": 1.504805, + "samples": 65, + "stdev": 0.19095214820511971, + "sum": 2.380261 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 7.0, + "mean": 0.21333333333333332, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 4.622222222222222, + "non_zero_median": 4.066666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 5.122666666666665, + "percentile999": 6.812266666666687, + "range": 7.0, + "samples": 65, + "stdev": 1.0489545059513095, + "sum": 13.866666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9999.866666666667, + "mean": 461.3764102564102, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9996.488888888889, + "non_zero_median": 9996.666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9997.818666666666, + "percentile999": 9999.661866666667, + "range": 9999.866666666667, + "samples": 65, + "stdev": 2113.767969778099, + "sum": 29989.466666666667 + }, + "pg_stat_database_blks_hit": { + "iqr": 4249.0, + "max": 38000.066666666666, + "mean": 21749.42358974359, + "median": 23146.133333333335, + "min": 6172.466666666666, + "non_zero_mean": 21749.42358974359, + "non_zero_median": 23146.133333333335, + "percentile25": 19845.2, + "percentile75": 24094.2, + "percentile90": 25835.706666666665, + "percentile99": 37855.64, + "percentile999": 37985.624, + "range": 31827.6, + "samples": 65, + "stdev": 6078.862945125132, + "sum": 1413712.5333333332 + }, + "pg_stat_database_blks_read": { + "iqr": 69.93333333333334, + "max": 154.86666666666667, + "mean": 98.61948717948718, + "median": 129.93333333333334, + "min": 0.0, + "non_zero_mean": 101.75026455026455, + "non_zero_median": 130.13333333333333, + "percentile25": 65.53333333333333, + "percentile75": 135.46666666666667, + "percentile90": 138.4, + "percentile99": 151.66666666666669, + "percentile999": 154.5466666666667, + "range": 154.86666666666667, + "samples": 65, + "stdev": 50.62358739191989, + "sum": 6410.266666666666 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 65, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 69.93333333333334, + "max": 154.86666666666667, + "mean": 98.61948717948718, + "median": 129.93333333333334, + "min": 0.0, + "non_zero_mean": 101.75026455026455, + "non_zero_median": 130.13333333333333, + "percentile25": 65.53333333333333, + "percentile75": 135.46666666666667, + "percentile90": 138.4, + "percentile99": 151.66666666666669, + "percentile999": 154.5466666666667, + "range": 154.86666666666667, + "samples": 65, + "stdev": 50.62358739191989, + "sum": 6410.266666666666 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4, + "mean": 0.21025641025641026, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8541666666666666, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.3573333333333333, + "percentile999": 1.3957333333333337, + "range": 1.4, + "samples": 65, + "stdev": 0.41753755141017773, + "sum": 13.666666666666664 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2049.466666666667, + "max": 24519.333333333332, + "mean": 7239.883076923077, + "median": 5731.666666666667, + "min": 3179.6, + "non_zero_mean": 7239.883076923077, + "non_zero_median": 5731.666666666667, + "percentile25": 5096.066666666667, + "percentile75": 7145.533333333334, + "percentile90": 11675.706666666667, + "percentile99": 22631.07733333333, + "percentile999": 24330.507733333354, + "range": 21339.733333333334, + "samples": 65, + "stdev": 4105.579901352934, + "sum": 470592.39999999997 + }, + "pg_stat_database_tup_inserted": { + "iqr": 364.6, + "max": 822.3333333333334, + "mean": 524.2646153846154, + "median": 687.1333333333333, + "min": 0.0, + "non_zero_mean": 540.9079365079365, + "non_zero_median": 687.2666666666667, + "percentile25": 359.26666666666665, + "percentile75": 723.8666666666667, + "percentile90": 736.72, + "percentile99": 804.6266666666667, + "percentile999": 820.5626666666669, + "range": 822.3333333333334, + "samples": 65, + "stdev": 268.1378793770802, + "sum": 34077.200000000004 + }, + "pg_stat_database_tup_returned": { + "iqr": 2401.2666666666655, + "max": 32592.533333333333, + "mean": 9095.350769230768, + "median": 6971.066666666667, + "min": 4379.0, + "non_zero_mean": 9095.350769230768, + "non_zero_median": 6971.066666666667, + "percentile25": 6216.533333333334, + "percentile75": 8617.8, + "percentile90": 16355.813333333334, + "percentile99": 27696.063999999995, + "percentile999": 32102.886400000054, + "range": 28213.533333333333, + "samples": 65, + "stdev": 5313.490146542061, + "sum": 591197.8000000002 + }, + "pg_stat_database_tup_updated": { + "iqr": 6.666666666666667, + "max": 137.8, + "mean": 10.064615384615385, + "median": 0.6666666666666666, + "min": 0.0, + "non_zero_mean": 11.682142857142857, + "non_zero_median": 1.4666666666666666, + "percentile25": 0.26666666666666666, + "percentile75": 6.933333333333334, + "percentile90": 24.773333333333344, + "percentile99": 118.64266666666666, + "percentile999": 135.8842666666669, + "range": 137.8, + "samples": 65, + "stdev": 25.928534177916514, + "sum": 654.1999999999998 + }, + "pg_stat_database_xact_commit": { + "iqr": 13.533333333333331, + "max": 143.73333333333332, + "mean": 58.76512820512821, + "median": 50.93333333333333, + "min": 5.133333333333334, + "non_zero_mean": 58.76512820512821, + "non_zero_median": 50.93333333333333, + "percentile25": 46.86666666666667, + "percentile75": 60.4, + "percentile90": 96.94666666666669, + "percentile99": 143.0933333333333, + "percentile999": 143.66933333333333, + "range": 138.6, + "samples": 65, + "stdev": 26.304256382687456, + "sum": 3819.733333333334 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 65, + "stdev": 0.0, + "sum": 4.3333333333333375 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.0041025641025641026, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.09599999999999985, + "percentile999": 0.24960000000000188, + "range": 0.26666666666666666, + "samples": 65, + "stdev": 0.03307592922378892, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 24.66666666666667, + "max": 151.73333333333332, + "mean": 78.82871794871795, + "median": 79.19999999999999, + "min": 8.6, + "non_zero_mean": 78.82871794871795, + "non_zero_median": 79.19999999999999, + "percentile25": 68.53333333333333, + "percentile75": 93.2, + "percentile90": 109.26666666666668, + "percentile99": 139.61599999999999, + "percentile999": 150.52160000000012, + "range": 143.13333333333333, + "samples": 65, + "stdev": 27.72184499940327, + "sum": 5123.866666666667 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,8,10,9,11,14,18,19,21,20,22,24,23,25,26,27,29,28,30,32,31,33,34,35,36,37,39,38,40,41,42,45,43,44,46,47,48,49,53,52,51,50,55,54,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-10T16:43:15.502730Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T16:43:16.307490Z 2021-12-10T16:43:16.303577Z 2021-12-10T16:43:16.184362Z 2021-12-10T16:43:15.617260Z 2021-12-10T16:43:15.609481Z 2021-12-10T16:43:15.518187Z 2021-12-10T16:43:15.502730Z 2021-12-10T16:43:17.312761Z 2021-12-10T16:43:16.460418Z 2021-12-10T16:43:16.322543Z 2021-12-10T16:47:33.447868Z 2021-12-10T16:47:33.348940Z 2021-12-10T16:47:32.593899Z 2021-12-10T16:47:32.561372Z 2021-12-10T16:47:32.542864Z 2021-12-10T16:47:32.510844Z 2021-12-10T16:47:32.481301Z 2021-12-10T16:47:32.475982Z 2021-12-10T16:47:32.391828Z 2021-12-10T16:47:32.301138Z 2021-12-10T16:50:34.252732Z 2021-12-10T16:50:34.157800Z 2021-12-10T16:50:33.660836Z 2021-12-10T16:50:33.645723Z 2021-12-10T16:50:33.637724Z 2021-12-10T16:50:33.628683Z 2021-12-10T16:50:33.608088Z 2021-12-10T16:50:33.538269Z 2021-12-10T16:50:33.502706Z 2021-12-10T16:50:33.462603Z 2021-12-10T16:53:42.415735Z 2021-12-10T16:53:42.282859Z 2021-12-10T16:53:41.867400Z 2021-12-10T16:53:41.860895Z 2021-12-10T16:53:41.857724Z 2021-12-10T16:53:41.824502Z 2021-12-10T16:53:41.786234Z 2021-12-10T16:53:41.699274Z 2021-12-10T16:53:41.673855Z 2021-12-10T16:53:41.631968Z 2021-12-10T16:56:49.413549Z 2021-12-10T16:56:49.413319Z 2021-12-10T16:56:49.021212Z 2021-12-10T16:56:49.019134Z 2021-12-10T16:56:48.924916Z 2021-12-10T16:56:48.897573Z 2021-12-10T16:56:48.860754Z 2021-12-10T16:56:48.856543Z 2021-12-10T16:56:48.814249Z 2021-12-10T16:56:48.809460Z", + "created_diff": 1.0269492, + "ended": "2021-12-10T16:59:25.321926Z", + "error_job_ids": " []", + "event_first": "2021-12-10T16:43:26.914739Z 2021-12-10T16:44:18.123307Z 2021-12-10T16:43:26.784504Z 2021-12-10T16:44:29.759729Z 2021-12-10T16:43:22.430514Z 2021-12-10T16:44:06.098500Z 2021-12-10T16:43:21.971758Z 2021-12-10T16:45:03.368475Z 2021-12-10T16:43:31.586974Z 2021-12-10T16:44:46.264430Z 2021-12-10T16:47:46.440319Z 2021-12-10T16:47:45.428513Z 2021-12-10T16:47:47.019778Z 2021-12-10T16:47:49.546629Z 2021-12-10T16:47:43.645469Z 2021-12-10T16:47:40.941608Z 2021-12-10T16:47:39.391675Z 2021-12-10T16:47:38.519009Z 2021-12-10T16:47:38.213003Z 2021-12-10T16:47:38.458872Z 2021-12-10T16:50:53.140379Z 2021-12-10T16:50:55.270014Z 2021-12-10T16:50:43.361413Z 2021-12-10T16:50:41.059469Z 2021-12-10T16:50:41.026594Z 2021-12-10T16:50:40.493137Z 2021-12-10T16:50:40.271435Z 2021-12-10T16:50:41.689137Z 2021-12-10T16:50:39.773452Z 2021-12-10T16:50:39.465280Z 2021-12-10T16:53:56.794043Z 2021-12-10T16:53:56.456200Z 2021-12-10T16:53:53.628255Z 2021-12-10T16:53:52.889542Z 2021-12-10T16:53:48.403394Z 2021-12-10T16:53:49.709871Z 2021-12-10T16:53:48.558358Z 2021-12-10T16:53:49.002952Z 2021-12-10T16:53:48.383445Z 2021-12-10T16:53:49.244916Z 2021-12-10T16:57:01.677983Z 2021-12-10T16:57:04.402578Z 2021-12-10T16:57:01.664684Z 2021-12-10T16:57:00.885668Z 2021-12-10T16:56:56.131135Z 2021-12-10T16:56:55.846912Z 2021-12-10T16:56:55.130418Z 2021-12-10T16:56:55.354967Z 2021-12-10T16:56:57.165450Z 2021-12-10T16:56:56.319781Z", + "event_last": "2021-12-10T16:46:00.848731Z 2021-12-10T16:46:44.200048Z 2021-12-10T16:46:00.116901Z 2021-12-10T16:46:50.537138Z 2021-12-10T16:45:55.530063Z 2021-12-10T16:46:35.507205Z 2021-12-10T16:45:55.162497Z 2021-12-10T16:47:13.319736Z 2021-12-10T16:46:06.150620Z 2021-12-10T16:47:02.418923Z 2021-12-10T16:50:09.374992Z 2021-12-10T16:50:09.107852Z 2021-12-10T16:50:10.246235Z 2021-12-10T16:50:12.875614Z 2021-12-10T16:50:06.844217Z 2021-12-10T16:50:07.578903Z 2021-12-10T16:50:03.960523Z 2021-12-10T16:50:04.431677Z 2021-12-10T16:50:03.240835Z 2021-12-10T16:50:03.459095Z 2021-12-10T16:53:16.106881Z 2021-12-10T16:53:16.581793Z 2021-12-10T16:53:07.334389Z 2021-12-10T16:53:08.456450Z 2021-12-10T16:53:05.432665Z 2021-12-10T16:53:07.217315Z 2021-12-10T16:53:05.951446Z 2021-12-10T16:53:07.585004Z 2021-12-10T16:53:06.235288Z 2021-12-10T16:53:07.907956Z 2021-12-10T16:56:19.217692Z 2021-12-10T16:56:20.220935Z 2021-12-10T16:56:18.373833Z 2021-12-10T16:56:18.006506Z 2021-12-10T16:56:13.723423Z 2021-12-10T16:56:16.513963Z 2021-12-10T16:56:15.136713Z 2021-12-10T16:56:14.895343Z 2021-12-10T16:56:15.136713Z 2021-12-10T16:56:14.300065Z 2021-12-10T16:59:23.485135Z 2021-12-10T16:59:25.321926Z 2021-12-10T16:59:23.493481Z 2021-12-10T16:59:22.570521Z 2021-12-10T16:59:18.978257Z 2021-12-10T16:59:19.125339Z 2021-12-10T16:59:19.088466Z 2021-12-10T16:59:18.530172Z 2021-12-10T16:59:20.080261Z 2021-12-10T16:59:19.487370Z", + "failed_job_ids": " []", + "finished": "2021-12-10T16:46:05.160732Z 2021-12-10T16:46:48.579197Z 2021-12-10T16:46:04.022398Z 2021-12-10T16:46:54.026079Z 2021-12-10T16:45:59.540926Z 2021-12-10T16:46:39.465280Z 2021-12-10T16:45:59.341281Z 2021-12-10T16:47:15.678875Z 2021-12-10T16:46:09.686226Z 2021-12-10T16:47:06.442754Z 2021-12-10T16:50:13.532802Z 2021-12-10T16:50:13.565496Z 2021-12-10T16:50:13.549806Z 2021-12-10T16:50:15.158720Z 2021-12-10T16:50:10.995625Z 2021-12-10T16:50:11.626409Z 2021-12-10T16:50:08.164111Z 2021-12-10T16:50:08.447438Z 2021-12-10T16:50:07.124997Z 2021-12-10T16:50:06.946705Z 2021-12-10T16:53:18.367954Z 2021-12-10T16:53:19.951560Z 2021-12-10T16:53:11.259566Z 2021-12-10T16:53:11.967049Z 2021-12-10T16:53:09.135362Z 2021-12-10T16:53:10.788781Z 2021-12-10T16:53:09.771754Z 2021-12-10T16:53:11.647035Z 2021-12-10T16:53:10.534222Z 2021-12-10T16:53:11.389555Z 2021-12-10T16:56:23.440406Z 2021-12-10T16:56:24.469977Z 2021-12-10T16:56:22.096418Z 2021-12-10T16:56:21.565422Z 2021-12-10T16:56:17.845435Z 2021-12-10T16:56:20.200215Z 2021-12-10T16:56:18.800449Z 2021-12-10T16:56:18.982420Z 2021-12-10T16:56:19.353623Z 2021-12-10T16:56:18.633307Z 2021-12-10T16:59:27.311520Z 2021-12-10T16:59:29.397769Z 2021-12-10T16:59:27.012941Z 2021-12-10T16:59:26.107066Z 2021-12-10T16:59:22.992587Z 2021-12-10T16:59:22.875498Z 2021-12-10T16:59:22.624719Z 2021-12-10T16:59:22.713173Z 2021-12-10T16:59:23.860548Z 2021-12-10T16:59:23.814462Z", + "finished_diff": 21.7526798, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 235.50848, + "mean": 163.35538556000003, + "min": 152.674986 + }, + "jobs_events_duration": { + "max": 154.563646, + "mean": 144.90596878000005, + "min": 129.951261 + }, + "jobs_events_lag": { + "max": -2.261073, + "mean": -3.8111908799999994, + "min": -4.457644 + }, + "jobs_waiting": { + "max": 10.180387, + "mean": 1.76752416, + "min": 0.89677 + }, + "modified": "2021-12-10T16:43:17.429639Z 2021-12-10T16:43:17.393235Z 2021-12-10T16:43:17.358621Z 2021-12-10T16:43:16.253972Z 2021-12-10T16:43:16.221253Z 2021-12-10T16:43:16.191003Z 2021-12-10T16:43:16.159831Z 2021-12-10T16:43:19.918421Z 2021-12-10T16:43:17.531585Z 2021-12-10T16:43:17.499150Z 2021-12-10T16:47:34.557377Z 2021-12-10T16:47:34.523201Z 2021-12-10T16:47:34.477441Z 2021-12-10T16:47:34.442214Z 2021-12-10T16:47:34.404666Z 2021-12-10T16:47:33.035774Z 2021-12-10T16:47:32.989157Z 2021-12-10T16:47:32.957259Z 2021-12-10T16:47:32.925074Z 2021-12-10T16:47:32.889262Z 2021-12-10T16:50:43.575911Z 2021-12-10T16:50:43.483669Z 2021-12-10T16:50:34.314991Z 2021-12-10T16:50:34.282416Z 2021-12-10T16:50:34.249250Z 2021-12-10T16:50:34.216458Z 2021-12-10T16:50:34.186936Z 2021-12-10T16:50:34.158335Z 2021-12-10T16:50:34.125770Z 2021-12-10T16:50:34.093995Z 2021-12-10T16:53:43.595328Z 2021-12-10T16:53:43.504364Z 2021-12-10T16:53:42.453662Z 2021-12-10T16:53:42.424233Z 2021-12-10T16:53:42.394076Z 2021-12-10T16:53:42.358370Z 2021-12-10T16:53:42.329436Z 2021-12-10T16:53:42.300457Z 2021-12-10T16:53:42.272048Z 2021-12-10T16:53:42.237438Z 2021-12-10T16:56:50.526099Z 2021-12-10T16:56:50.442084Z 2021-12-10T16:56:50.382573Z 2021-12-10T16:56:50.316480Z 2021-12-10T16:56:49.542088Z 2021-12-10T16:56:49.506819Z 2021-12-10T16:56:49.473529Z 2021-12-10T16:56:49.440599Z 2021-12-10T16:56:49.407679Z 2021-12-10T16:56:49.369926Z", + "modified_diff": 3.4845368, + "started": "2021-12-10T16:43:17.955156Z 2021-12-10T16:43:17.881639Z 2021-12-10T16:43:17.803904Z 2021-12-10T16:43:16.747811Z 2021-12-10T16:43:16.627335Z 2021-12-10T16:43:16.626295Z 2021-12-10T16:43:16.446519Z 2021-12-10T16:43:20.170395Z 2021-12-10T16:43:18.208469Z 2021-12-10T16:43:18.071631Z 2021-12-10T16:47:35.400954Z 2021-12-10T16:47:35.238873Z 2021-12-10T16:47:35.081217Z 2021-12-10T16:47:34.952215Z 2021-12-10T16:47:34.844183Z 2021-12-10T16:47:33.554556Z 2021-12-10T16:47:33.471865Z 2021-12-10T16:47:33.386515Z 2021-12-10T16:47:33.288598Z 2021-12-10T16:47:33.212140Z 2021-12-10T16:50:44.240098Z 2021-12-10T16:50:44.338187Z 2021-12-10T16:50:35.221104Z 2021-12-10T16:50:35.091073Z 2021-12-10T16:50:35.012366Z 2021-12-10T16:50:34.913786Z 2021-12-10T16:50:34.832864Z 2021-12-10T16:50:34.742843Z 2021-12-10T16:50:34.667284Z 2021-12-10T16:50:34.578975Z 2021-12-10T16:53:44.261435Z 2021-12-10T16:53:44.150798Z 2021-12-10T16:53:43.226467Z 2021-12-10T16:53:43.151576Z 2021-12-10T16:53:43.048006Z 2021-12-10T16:53:42.954667Z 2021-12-10T16:53:42.864636Z 2021-12-10T16:53:42.782444Z 2021-12-10T16:53:42.692670Z 2021-12-10T16:53:42.620996Z 2021-12-10T16:56:51.294129Z 2021-12-10T16:56:51.179821Z 2021-12-10T16:56:50.951856Z 2021-12-10T16:56:50.907280Z 2021-12-10T16:56:50.128393Z 2021-12-10T16:56:50.034675Z 2021-12-10T16:56:49.949733Z 2021-12-10T16:56:49.872802Z 2021-12-10T16:56:49.794393Z 2021-12-10T16:56:49.721745Z", + "started_diff": 3.776945, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-10T17:02:10.434901+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-launching.json new file mode 100644 index 00000000..5665713e --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-10T14_50_06_287_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-10T17:47:41.303900+00:00", + "golden": "true", + "id": "run-2021-12-10T14_50_06_287_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17821.333333333336, + "max": 27769.066666666666, + "mean": 9897.024581386917, + "median": 4169.433333333334, + "min": 1335.2666666666667, + "non_zero_mean": 9897.024581386917, + "non_zero_median": 4169.433333333334, + "percentile25": 1740.1, + "percentile75": 19561.433333333334, + "percentile90": 23181.266666666666, + "percentile99": 27074.466666666667, + "percentile999": 27699.606666666677, + "range": 26433.8, + "samples": 76, + "stdev": 9322.032115439448, + "sum": 752173.8681854056 + } + }, + "cpu": { + "system": { + "iqr": 1.351833333333333, + "max": 1.7340000000000013, + "mean": 0.6633859666687331, + "median": 0.24100000000000157, + "min": 0.026666666666667043, + "non_zero_mean": 0.6633859666687331, + "non_zero_median": 0.24100000000000157, + "percentile25": 0.03666666666667122, + "percentile75": 1.3885000000000043, + "percentile90": 1.6069999999999984, + "percentile99": 1.7305000000000015, + "percentile999": 1.7336500000000015, + "range": 1.7073333333333343, + "samples": 76, + "stdev": 0.6845808543443203, + "sum": 50.41733346682372 + }, + "user": { + "iqr": 5.80733333333334, + "max": 6.5106666666666655, + "mean": 2.877491222811735, + "median": 1.4389999999999987, + "min": 0.08599999999998997, + "non_zero_mean": 2.877491222811735, + "non_zero_median": 1.4389999999999987, + "percentile25": 0.3029999999999954, + "percentile75": 6.110333333333336, + "percentile90": 6.347333333333329, + "percentile99": 6.497166666666661, + "percentile999": 6.509316666666665, + "range": 6.424666666666676, + "samples": 76, + "stdev": 2.77194324215036, + "sum": 218.68933293369184 + } + }, + "entropy_available_bits": { + "iqr": 4.283337777481502, + "max": 421.33333333333337, + "mean": 24.250877193013647, + "median": 1.4333333333333331, + "min": 0.0, + "non_zero_mean": 30.214207650312083, + "non_zero_median": 3.6666666666666665, + "percentile25": 0.5999955558518321, + "percentile75": 4.883333333333334, + "percentile90": 6.199999999999999, + "percentile99": 265.43333333333334, + "percentile999": 405.7433333333357, + "range": 421.33333333333337, + "samples": 76, + "stdev": 73.21630813531202, + "sum": 1843.0666666690372 + }, + "memory": { + "used": { + "iqr": 5053538304.0, + "max": 17433976832.0, + "mean": 8897344242.526316, + "median": 6643429376.0, + "min": 5197103104.0, + "non_zero_mean": 8897344242.526316, + "non_zero_median": 6643429376.0, + "percentile25": 6321975296.0, + "percentile75": 11375513600.0, + "percentile90": 15640121344.0, + "percentile99": 16857841664.0, + "percentile999": 17376363315.20001, + "range": 12236873728.0, + "samples": 76, + "stdev": 3809751550.39947, + "sum": 676198162432.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.007017543859649123, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.26666666666666666, + "percentile999": 0.26666666666666666, + "range": 0.26666666666666666, + "samples": 76, + "stdev": 0.042969684222521766, + "sum": 0.5333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 928.1500000000001, + "max": 1769.5333333333333, + "mean": 465.8692967252476, + "median": 49.63333333333334, + "min": 0.26666666666666666, + "non_zero_mean": 465.8692967252476, + "non_zero_median": 49.63333333333334, + "percentile25": 2.7666666666666666, + "percentile75": 930.9166666666667, + "percentile90": 1494.7666666666667, + "percentile99": 1709.7333333333336, + "percentile999": 1763.5533333333342, + "range": 1769.2666666666667, + "samples": 76, + "stdev": 594.062069382903, + "sum": 35406.06655111882 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2286.083333333333, + "max": 3760.3333333333335, + "mean": 1715.9850877192982, + "median": 997.1000000000001, + "min": 402.93333333333334, + "non_zero_mean": 1715.9850877192982, + "non_zero_median": 997.1000000000001, + "percentile25": 603.3000000000001, + "percentile75": 2889.383333333333, + "percentile90": 3150.2666666666664, + "percentile99": 3689.733333333333, + "percentile999": 3753.2733333333344, + "range": 3357.4, + "samples": 76, + "stdev": 1159.315349811255, + "sum": 130414.86666666658 + } + }, + "cpu": { + "system": { + "iqr": 0.20050000000000023, + "max": 0.402000000000001, + "mean": 0.13248245614035087, + "median": 0.05433333333333295, + "min": 0.01199999999999856, + "non_zero_mean": 0.13248245614035087, + "non_zero_median": 0.05433333333333295, + "percentile25": 0.02466666666666685, + "percentile75": 0.2251666666666671, + "percentile90": 0.31066666666666665, + "percentile99": 0.39500000000000096, + "percentile999": 0.4013000000000011, + "range": 0.39000000000000246, + "samples": 76, + "stdev": 0.12139063841794634, + "sum": 10.068666666666667 + }, + "user": { + "iqr": 0.30549999999999855, + "max": 0.8159999999999987, + "mean": 0.20218421052631577, + "median": 0.086666666666666, + "min": 0.010666666666666439, + "non_zero_mean": 0.20218421052631577, + "non_zero_median": 0.086666666666666, + "percentile25": 0.0386666666666675, + "percentile75": 0.34416666666666607, + "percentile90": 0.4296666666666667, + "percentile99": 0.7305000000000005, + "percentile999": 0.8074500000000002, + "range": 0.8053333333333322, + "samples": 76, + "stdev": 0.19936413284712723, + "sum": 15.366 + } + }, + "entropy_available_bits": { + "iqr": 3.45, + "max": 210.6, + "mean": 12.280701754385964, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 18.666666666666668, + "non_zero_median": 1.8333333333333333, + "percentile25": 0.0, + "percentile75": 3.45, + "percentile90": 4.733333333333333, + "percentile99": 210.6, + "percentile999": 210.6, + "range": 210.6, + "samples": 76, + "stdev": 46.9132258894315, + "sum": 933.3333333333331 + }, + "memory": { + "used": { + "iqr": 454222848.0, + "max": 1106575360.0, + "mean": 577924904.4210526, + "median": 400220160.0, + "min": 360714240.0, + "non_zero_mean": 577924904.4210526, + "non_zero_median": 400220160.0, + "percentile25": 372633600.0, + "percentile75": 826856448.0, + "percentile90": 1022062592.0, + "percentile99": 1101921280.0, + "percentile999": 1106109952.0, + "range": 745861120.0, + "samples": 76, + "stdev": 268913608.3423139, + "sum": 43922292736.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1092.2666666666667, + "max": 169847.46666666667, + "mean": 17332.547368421052, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 65863.68000000001, + "non_zero_median": 47786.66666666667, + "percentile25": 0.0, + "percentile75": 1092.2666666666667, + "percentile90": 62532.26666666666, + "percentile99": 164113.06666666668, + "percentile999": 169274.02666666676, + "range": 169847.46666666667, + "samples": 76, + "stdev": 39439.48603106819, + "sum": 1317273.6000000003 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.40412, + "mean": 0.02080207894736842, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.1053972, + "non_zero_median": 0.012028, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.011355, + "percentile99": 0.39448099999999997, + "percentile999": 0.40315610000000013, + "range": 0.40412, + "samples": 76, + "stdev": 0.08219315544719656, + "sum": 1.5809580000000003 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.193439, + "max": 7.860336, + "mean": 0.4192971447368421, + "median": 0.0, + "min": -0.000629, + "non_zero_mean": 0.8385942894736842, + "non_zero_median": 0.19674999999999998, + "percentile25": 0.0, + "percentile75": 0.193439, + "percentile90": 1.3826044999999998, + "percentile99": 4.78859925, + "percentile999": 7.553162325000047, + "range": 7.860965, + "samples": 76, + "stdev": 1.103872625741922, + "sum": 31.866583 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 2.8666666666666667, + "mean": 0.08771929824561404, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.2222222222222223, + "non_zero_median": 2.066666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.2666666666666666, + "percentile999": 2.806666666666676, + "range": 2.8666666666666667, + "samples": 76, + "stdev": 0.44584807594360226, + "sum": 6.666666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 7720.6, + "mean": 273.45438596491226, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6927.511111111111, + "non_zero_median": 6672.466666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 6934.5, + "percentile999": 7641.9900000000125, + "range": 7720.6, + "samples": 76, + "stdev": 1362.7016556373603, + "sum": 20782.533333333333 + }, + "pg_stat_database_blks_hit": { + "iqr": 23075.449999999997, + "max": 104802.26666666666, + "mean": 22054.88245614035, + "median": 11344.400000000001, + "min": 1206.1333333333334, + "non_zero_mean": 22054.88245614035, + "non_zero_median": 11344.400000000001, + "percentile25": 7444.466666666667, + "percentile75": 30519.916666666664, + "percentile90": 47221.46666666667, + "percentile99": 92209.16666666666, + "percentile999": 103542.95666666685, + "range": 103596.13333333333, + "samples": 76, + "stdev": 22342.07610948983, + "sum": 1676171.0666666662 + }, + "pg_stat_database_blks_read": { + "iqr": 2.6166666666666667, + "max": 13.6, + "mean": 2.180701754385965, + "median": 0.7666666666666666, + "min": 0.0, + "non_zero_mean": 3.682962962962963, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 2.6166666666666667, + "percentile90": 6.833333333333333, + "percentile99": 13.55, + "percentile999": 13.595, + "range": 13.6, + "samples": 76, + "stdev": 3.221240907236859, + "sum": 165.73333333333332 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 76, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 2.6166666666666667, + "max": 13.6, + "mean": 2.180701754385965, + "median": 0.7666666666666666, + "min": 0.0, + "non_zero_mean": 3.682962962962963, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 2.6166666666666667, + "percentile90": 6.833333333333333, + "percentile99": 13.55, + "percentile999": 13.595, + "range": 13.6, + "samples": 76, + "stdev": 3.221240907236859, + "sum": 165.73333333333332 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 2.3333333333333335, + "mean": 0.22719298245614036, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.3282051282051281, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9333333333333333, + "percentile99": 2.1833333333333336, + "percentile999": 2.3183333333333356, + "range": 2.3333333333333335, + "samples": 76, + "stdev": 0.584255411583763, + "sum": 17.26666666666667 + }, + "pg_stat_database_tup_fetched": { + "iqr": 13125.550000000001, + "max": 52588.8, + "mean": 11763.084210526316, + "median": 6078.166666666666, + "min": 577.4, + "non_zero_mean": 11763.084210526316, + "non_zero_median": 6078.166666666666, + "percentile25": 3845.1666666666665, + "percentile75": 16970.716666666667, + "percentile90": 24391.666666666664, + "percentile99": 46177.25, + "percentile999": 51947.6450000001, + "range": 52011.4, + "samples": 76, + "stdev": 11550.143879073668, + "sum": 893994.4 + }, + "pg_stat_database_tup_inserted": { + "iqr": 26.616666666666667, + "max": 111.0, + "mean": 16.905263157894737, + "median": 5.133333333333333, + "min": 0.0, + "non_zero_mean": 26.220408163265304, + "non_zero_median": 20.2, + "percentile25": 0.0, + "percentile75": 26.616666666666667, + "percentile90": 47.233333333333334, + "percentile99": 106.39999999999999, + "percentile999": 110.54000000000006, + "range": 111.0, + "samples": 76, + "stdev": 24.912854960846918, + "sum": 1284.7999999999997 + }, + "pg_stat_database_tup_returned": { + "iqr": 17786.88333333333, + "max": 77991.2, + "mean": 17429.505263157895, + "median": 9229.433333333334, + "min": 1617.2666666666667, + "non_zero_mean": 17429.505263157895, + "non_zero_median": 9229.433333333334, + "percentile25": 6512.15, + "percentile75": 24299.033333333333, + "percentile90": 35460.7, + "percentile99": 72579.0, + "percentile999": 77449.98000000008, + "range": 76373.93333333333, + "samples": 76, + "stdev": 16924.51261026164, + "sum": 1324642.4000000001 + }, + "pg_stat_database_tup_updated": { + "iqr": 29.15, + "max": 85.53333333333333, + "mean": 17.171929824561403, + "median": 7.8999999999999995, + "min": 0.0, + "non_zero_mean": 19.192156862745097, + "non_zero_median": 13.833333333333334, + "percentile25": 0.26666666666666666, + "percentile75": 29.416666666666664, + "percentile90": 44.266666666666666, + "percentile99": 81.68333333333334, + "percentile999": 85.1483333333334, + "range": 85.53333333333333, + "samples": 76, + "stdev": 20.511541865681437, + "sum": 1305.0666666666668 + }, + "pg_stat_database_xact_commit": { + "iqr": 287.25000000000006, + "max": 595.0666666666667, + "mean": 156.00438596491227, + "median": 67.80000000000001, + "min": 2.466666666666667, + "non_zero_mean": 156.00438596491227, + "non_zero_median": 67.80000000000001, + "percentile25": 9.45, + "percentile75": 296.70000000000005, + "percentile90": 373.0, + "percentile99": 525.7166666666667, + "percentile999": 588.1316666666678, + "range": 592.6, + "samples": 76, + "stdev": 164.4578506254982, + "sum": 11856.333333333334 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 76, + "stdev": 0.0, + "sum": 5.066666666666668 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.0035087719298245615, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06666666666666667, + "percentile999": 0.2466666666666697, + "range": 0.26666666666666666, + "samples": 76, + "stdev": 0.030588764516074902, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 100.95, + "max": 160.8, + "mean": 55.3859649122807, + "median": 35.63333333333334, + "min": 0.26666666666666666, + "non_zero_mean": 55.3859649122807, + "non_zero_median": 35.63333333333334, + "percentile25": 5.300000000000001, + "percentile75": 106.25, + "percentile90": 131.23333333333335, + "percentile99": 155.14999999999998, + "percentile999": 160.2350000000001, + "range": 160.53333333333333, + "samples": 76, + "stdev": 52.6079507476897, + "sum": 4209.333333333333 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,60,64,62,63,66,72,73,68,71,75,76,77,70,65,78,79,87,81,74,69,82,86,93,84,83,91,94,88,92,98,89,106,99,95,96,101,97,102,100,105,109,104,107,110,103,114,112,115,113,111,118,116,122,119,125,124,120,126,128,121,130,123,131,129,108,117,127,136,134,133,138,139,135,142,141,146,147,150,148,140,151,152,144,145,154,157,158,155,160,163,162,159,164,165,156,166,167,168,169,172,170,176,173,171,174,177,181,175,191,178,187,179,182,184,188,186,185,180,189,192,190,183,196,197,195,198,200,194,201,193,202,199,204,203,205,209,208,207,210,211,206,212,214,218,220,219,223,224,216,215,221,217,222,213,226,225,230,234,240,231,229,233,241,238,235,232,236,237,228,227,239,243,246,245,244,249,247,242,248,252,251,254,255,259,250,256,258,260,261,263,253,264,262,257,265,266,267,268,269,272,270,271,277,275,281,274,273,286,279,278,297,282,283,276,291,294,298,284,288,285,280,293,299,289,300,287,290,292,296,295,303,301,302,306,308,309,304,315,310,312,305,307,311,313,319,317,320,316,314,318,321,327,324,323,326,328,322,325,337,335,332,331,330,333,338,342,339,336,340,329,334,341,344,347,352,350,345,351,349,346,356,343,353,358,360,357,361,354,359,365,355,364,348,362,366,363,367,368,369,371,370,378,373,374,372,382,380,388,379,375,381,383,384,377,387,389,386,393,376,385,396,395,390,392,397,398,391,394,405,401,402,400,399,404,403,408,407,413,409,410,406,412,418,411,414,420,419,416,415,422,425,424,417,423,427,428,421,426,434,430,435,432,437,431,440,438,442,439,449,443,447,444,446,451,441,448,433,455,445,452,453,454,436,464,459,429,460,458,461,456,462,467,466,463,465,450,457,468,469,470,472,471,473,474,478,477,475,479,480,488,490,483,476,485,481,484,486,482,487,489,492,491,494,493,495,497,496,498,503,504,499,506,502,501,505,500,511,510,520,507,517,509,516,512,508,518,513,521,514,515,526,519,525,522,533,524,523,534,527,528,535,529,536,531,532,538,539,540,541,537,543,542,544,549,546,548,550,551,545,552,554,555,556,560,553,558,557,564,561,559,562,530,563,547,568,565,567,569,566", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-10T17:03:59.471762Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-10T17:04:03.905900Z 2021-12-10T17:04:03.760728Z 2021-12-10T17:04:03.528680Z 2021-12-10T17:04:03.456637Z 2021-12-10T17:04:03.336165Z 2021-12-10T17:04:03.202498Z 2021-12-10T17:04:03.180801Z 2021-12-10T17:04:03.161091Z 2021-12-10T17:04:02.886473Z 2021-12-10T17:04:02.641228Z 2021-12-10T17:04:02.429615Z 2021-12-10T17:04:02.382945Z 2021-12-10T17:04:02.379276Z 2021-12-10T17:04:02.238817Z 2021-12-10T17:04:02.194770Z 2021-12-10T17:04:02.012939Z 2021-12-10T17:04:01.755835Z 2021-12-10T17:04:01.582506Z 2021-12-10T17:04:01.472487Z 2021-12-10T17:04:01.158818Z 2021-12-10T17:04:01.021735Z 2021-12-10T17:04:01.014315Z 2021-12-10T17:04:00.818984Z 2021-12-10T17:04:00.809279Z 2021-12-10T17:04:00.722547Z 2021-12-10T17:04:00.696913Z 2021-12-10T17:04:00.657574Z 2021-12-10T17:04:00.640360Z 2021-12-10T17:04:00.407514Z 2021-12-10T17:04:00.337294Z 2021-12-10T17:04:00.269077Z 2021-12-10T17:03:59.919211Z 2021-12-10T17:03:59.705585Z 2021-12-10T17:03:59.592255Z 2021-12-10T17:03:59.565757Z 2021-12-10T17:03:59.471762Z 2021-12-10T17:04:18.539235Z 2021-12-10T17:04:17.410223Z 2021-12-10T17:04:16.241745Z 2021-12-10T17:04:15.380557Z 2021-12-10T17:04:14.540547Z 2021-12-10T17:04:14.478797Z 2021-12-10T17:04:13.530181Z 2021-12-10T17:04:13.188022Z 2021-12-10T17:04:11.422487Z 2021-12-10T17:04:10.977793Z 2021-12-10T17:04:10.970711Z 2021-12-10T17:04:10.883670Z 2021-12-10T17:04:10.816151Z 2021-12-10T17:04:10.702238Z 2021-12-10T17:04:10.610432Z 2021-12-10T17:04:10.531971Z 2021-12-10T17:04:10.424319Z 2021-12-10T17:04:10.319085Z 2021-12-10T17:04:10.131780Z 2021-12-10T17:04:09.854649Z 2021-12-10T17:04:09.852440Z 2021-12-10T17:04:09.777084Z 2021-12-10T17:04:09.456288Z 2021-12-10T17:04:09.385163Z 2021-12-10T17:04:09.378126Z 2021-12-10T17:04:09.355069Z 2021-12-10T17:04:09.109865Z 2021-12-10T17:04:09.084161Z 2021-12-10T17:04:08.474883Z 2021-12-10T17:04:08.399607Z 2021-12-10T17:04:08.342881Z 2021-12-10T17:04:07.997812Z 2021-12-10T17:04:07.823355Z 2021-12-10T17:04:07.672019Z 2021-12-10T17:04:07.415010Z 2021-12-10T17:04:07.377073Z 2021-12-10T17:04:07.258147Z 2021-12-10T17:04:07.044669Z 2021-12-10T17:04:06.784855Z 2021-12-10T17:04:06.683440Z 2021-12-10T17:04:06.661363Z 2021-12-10T17:04:06.542045Z 2021-12-10T17:04:06.444191Z 2021-12-10T17:04:06.238834Z 2021-12-10T17:04:06.220042Z 2021-12-10T17:04:06.138295Z 2021-12-10T17:04:06.012291Z 2021-12-10T17:04:05.959788Z 2021-12-10T17:04:05.932736Z 2021-12-10T17:04:05.615126Z 2021-12-10T17:04:05.612540Z 2021-12-10T17:04:05.565124Z 2021-12-10T17:04:05.540312Z 2021-12-10T17:04:04.936768Z 2021-12-10T17:04:04.778208Z 2021-12-10T17:04:04.748354Z 2021-12-10T17:04:04.684903Z 2021-12-10T17:04:04.677745Z 2021-12-10T17:04:04.642764Z 2021-12-10T17:04:04.632556Z 2021-12-10T17:04:04.232264Z 2021-12-10T17:04:04.115332Z 2021-12-10T17:04:04.073624Z 2021-12-10T17:04:03.928685Z 2021-12-10T17:08:40.348609Z 2021-12-10T17:08:39.997588Z 2021-12-10T17:08:37.714526Z 2021-12-10T17:08:36.847931Z 2021-12-10T17:08:36.375729Z 2021-12-10T17:08:36.205361Z 2021-12-10T17:08:35.424960Z 2021-12-10T17:08:35.297944Z 2021-12-10T17:08:34.637599Z 2021-12-10T17:08:34.476964Z 2021-12-10T17:08:33.754999Z 2021-12-10T17:08:33.361266Z 2021-12-10T17:08:33.160907Z 2021-12-10T17:08:33.003750Z 2021-12-10T17:08:32.152296Z 2021-12-10T17:08:32.075671Z 2021-12-10T17:08:31.879685Z 2021-12-10T17:08:31.761765Z 2021-12-10T17:08:31.709310Z 2021-12-10T17:08:31.575277Z 2021-12-10T17:08:31.398977Z 2021-12-10T17:08:31.039169Z 2021-12-10T17:08:30.998578Z 2021-12-10T17:08:30.808473Z 2021-12-10T17:08:30.648736Z 2021-12-10T17:08:30.445178Z 2021-12-10T17:08:30.373362Z 2021-12-10T17:08:30.263199Z 2021-12-10T17:08:28.606471Z 2021-12-10T17:08:28.507420Z 2021-12-10T17:08:28.362525Z 2021-12-10T17:08:28.244378Z 2021-12-10T17:08:28.194277Z 2021-12-10T17:08:28.116604Z 2021-12-10T17:08:27.861986Z 2021-12-10T17:08:27.718540Z 2021-12-10T17:08:27.705185Z 2021-12-10T17:08:27.573913Z 2021-12-10T17:08:27.355194Z 2021-12-10T17:08:27.163954Z 2021-12-10T17:08:27.146516Z 2021-12-10T17:08:27.044166Z 2021-12-10T17:08:26.868817Z 2021-12-10T17:08:26.389977Z 2021-12-10T17:08:26.229342Z 2021-12-10T17:08:25.795504Z 2021-12-10T17:08:25.493244Z 2021-12-10T17:08:25.426367Z 2021-12-10T17:08:25.389156Z 2021-12-10T17:08:25.145971Z 2021-12-10T17:08:25.006090Z 2021-12-10T17:08:24.895700Z 2021-12-10T17:08:24.877689Z 2021-12-10T17:08:24.875244Z 2021-12-10T17:08:24.812965Z 2021-12-10T17:08:24.634238Z 2021-12-10T17:08:24.389455Z 2021-12-10T17:08:24.074761Z 2021-12-10T17:08:23.965369Z 2021-12-10T17:08:23.941931Z 2021-12-10T17:08:23.923221Z 2021-12-10T17:08:23.918392Z 2021-12-10T17:08:23.906734Z 2021-12-10T17:08:23.777543Z 2021-12-10T17:08:23.250774Z 2021-12-10T17:08:23.001478Z 2021-12-10T17:08:22.665268Z 2021-12-10T17:08:22.432988Z 2021-12-10T17:08:22.373219Z 2021-12-10T17:08:22.360141Z 2021-12-10T17:08:22.291482Z 2021-12-10T17:08:22.219703Z 2021-12-10T17:08:22.154442Z 2021-12-10T17:08:22.122285Z 2021-12-10T17:08:21.989087Z 2021-12-10T17:08:21.986971Z 2021-12-10T17:08:21.678582Z 2021-12-10T17:08:21.488075Z 2021-12-10T17:08:21.408914Z 2021-12-10T17:08:21.307126Z 2021-12-10T17:08:21.181926Z 2021-12-10T17:08:21.164616Z 2021-12-10T17:08:21.163165Z 2021-12-10T17:08:21.132237Z 2021-12-10T17:08:20.794725Z 2021-12-10T17:08:20.722998Z 2021-12-10T17:08:20.475221Z 2021-12-10T17:08:20.362803Z 2021-12-10T17:08:20.294715Z 2021-12-10T17:08:20.278900Z 2021-12-10T17:08:20.229960Z 2021-12-10T17:08:20.072008Z 2021-12-10T17:08:19.991255Z 2021-12-10T17:08:19.670278Z 2021-12-10T17:08:19.658873Z 2021-12-10T17:08:19.196392Z 2021-12-10T17:08:18.990879Z 2021-12-10T17:08:18.965093Z 2021-12-10T17:08:18.889321Z 2021-12-10T17:08:18.874783Z 2021-12-10T17:12:44.082161Z 2021-12-10T17:12:43.224691Z 2021-12-10T17:12:41.769561Z 2021-12-10T17:12:40.821402Z 2021-12-10T17:12:40.784752Z 2021-12-10T17:12:40.612413Z 2021-12-10T17:12:40.553952Z 2021-12-10T17:12:40.192623Z 2021-12-10T17:12:40.086594Z 2021-12-10T17:12:39.958301Z 2021-12-10T17:12:39.930819Z 2021-12-10T17:12:39.798957Z 2021-12-10T17:12:39.712632Z 2021-12-10T17:12:39.499565Z 2021-12-10T17:12:39.396270Z 2021-12-10T17:12:39.329465Z 2021-12-10T17:12:39.257345Z 2021-12-10T17:12:39.125168Z 2021-12-10T17:12:38.985812Z 2021-12-10T17:12:38.619309Z 2021-12-10T17:12:38.590457Z 2021-12-10T17:12:38.493513Z 2021-12-10T17:12:38.234871Z 2021-12-10T17:12:38.055705Z 2021-12-10T17:12:37.850023Z 2021-12-10T17:12:37.508757Z 2021-12-10T17:12:37.430752Z 2021-12-10T17:12:37.196151Z 2021-12-10T17:12:36.608088Z 2021-12-10T17:12:36.402984Z 2021-12-10T17:12:36.276861Z 2021-12-10T17:12:36.245273Z 2021-12-10T17:12:36.236705Z 2021-12-10T17:12:36.233979Z 2021-12-10T17:12:36.224798Z 2021-12-10T17:12:36.180419Z 2021-12-10T17:12:36.170179Z 2021-12-10T17:12:36.109534Z 2021-12-10T17:12:35.953031Z 2021-12-10T17:12:35.736074Z 2021-12-10T17:12:35.598347Z 2021-12-10T17:12:35.529173Z 2021-12-10T17:12:35.103258Z 2021-12-10T17:12:35.090963Z 2021-12-10T17:12:35.031893Z 2021-12-10T17:12:34.979658Z 2021-12-10T17:12:34.779596Z 2021-12-10T17:12:34.610672Z 2021-12-10T17:12:34.514841Z 2021-12-10T17:12:34.249787Z 2021-12-10T17:12:34.127176Z 2021-12-10T17:12:33.709914Z 2021-12-10T17:12:33.634880Z 2021-12-10T17:12:33.587060Z 2021-12-10T17:12:33.576190Z 2021-12-10T17:12:33.500356Z 2021-12-10T17:12:33.449778Z 2021-12-10T17:12:33.382070Z 2021-12-10T17:12:33.132524Z 2021-12-10T17:12:33.114216Z 2021-12-10T17:12:33.059363Z 2021-12-10T17:12:32.930596Z 2021-12-10T17:12:32.516468Z 2021-12-10T17:12:32.445567Z 2021-12-10T17:12:32.397721Z 2021-12-10T17:12:32.394357Z 2021-12-10T17:12:32.205792Z 2021-12-10T17:12:31.926763Z 2021-12-10T17:12:31.875067Z 2021-12-10T17:12:31.866987Z 2021-12-10T17:12:31.837228Z 2021-12-10T17:12:31.657792Z 2021-12-10T17:12:31.642042Z 2021-12-10T17:12:31.623094Z 2021-12-10T17:12:31.576157Z 2021-12-10T17:12:31.493797Z 2021-12-10T17:12:31.437937Z 2021-12-10T17:12:31.377074Z 2021-12-10T17:12:31.369245Z 2021-12-10T17:12:31.170049Z 2021-12-10T17:12:31.167641Z 2021-12-10T17:12:30.892626Z 2021-12-10T17:12:30.690019Z 2021-12-10T17:12:30.558357Z 2021-12-10T17:12:30.531693Z 2021-12-10T17:12:30.375145Z 2021-12-10T17:12:30.328729Z 2021-12-10T17:12:30.229921Z 2021-12-10T17:12:30.119199Z 2021-12-10T17:12:30.007383Z 2021-12-10T17:12:29.924694Z 2021-12-10T17:12:29.908211Z 2021-12-10T17:12:29.663437Z 2021-12-10T17:12:29.608674Z 2021-12-10T17:12:29.116698Z 2021-12-10T17:12:29.115577Z 2021-12-10T17:12:29.115069Z 2021-12-10T17:12:29.090923Z 2021-12-10T17:12:29.088113Z 2021-12-10T17:12:28.794219Z 2021-12-10T17:16:56.293762Z 2021-12-10T17:16:55.812845Z 2021-12-10T17:16:55.546713Z 2021-12-10T17:16:55.488322Z 2021-12-10T17:16:55.310178Z 2021-12-10T17:16:55.267440Z 2021-12-10T17:16:55.247899Z 2021-12-10T17:16:55.109166Z 2021-12-10T17:16:54.883460Z 2021-12-10T17:16:54.854389Z 2021-12-10T17:16:54.821029Z 2021-12-10T17:16:54.745372Z 2021-12-10T17:16:54.586647Z 2021-12-10T17:16:54.491026Z 2021-12-10T17:16:54.183911Z 2021-12-10T17:16:53.999455Z 2021-12-10T17:16:53.972642Z 2021-12-10T17:16:53.910236Z 2021-12-10T17:16:53.753298Z 2021-12-10T17:16:53.665592Z 2021-12-10T17:16:53.491245Z 2021-12-10T17:16:53.409652Z 2021-12-10T17:16:53.343197Z 2021-12-10T17:16:53.314852Z 2021-12-10T17:16:53.227029Z 2021-12-10T17:16:53.123903Z 2021-12-10T17:16:52.913800Z 2021-12-10T17:16:52.851250Z 2021-12-10T17:16:52.837215Z 2021-12-10T17:16:52.734476Z 2021-12-10T17:16:52.647159Z 2021-12-10T17:16:52.582566Z 2021-12-10T17:16:52.141659Z 2021-12-10T17:16:52.057832Z 2021-12-10T17:16:51.974661Z 2021-12-10T17:16:51.842137Z 2021-12-10T17:16:51.840212Z 2021-12-10T17:16:51.808013Z 2021-12-10T17:16:51.763931Z 2021-12-10T17:16:51.566479Z 2021-12-10T17:16:51.460532Z 2021-12-10T17:16:51.157126Z 2021-12-10T17:16:51.091708Z 2021-12-10T17:16:51.068665Z 2021-12-10T17:16:50.994389Z 2021-12-10T17:16:50.787530Z 2021-12-10T17:16:50.663630Z 2021-12-10T17:16:50.655006Z 2021-12-10T17:16:50.517216Z 2021-12-10T17:16:50.486188Z 2021-12-10T17:16:50.221783Z 2021-12-10T17:16:50.122872Z 2021-12-10T17:16:50.096279Z 2021-12-10T17:16:49.994616Z 2021-12-10T17:16:50.017338Z 2021-12-10T17:16:49.791835Z 2021-12-10T17:16:49.598289Z 2021-12-10T17:16:49.510152Z 2021-12-10T17:16:49.508695Z 2021-12-10T17:16:49.507330Z 2021-12-10T17:16:49.318240Z 2021-12-10T17:16:49.295966Z 2021-12-10T17:16:49.273001Z 2021-12-10T17:16:49.061300Z 2021-12-10T17:16:48.747018Z 2021-12-10T17:16:48.742374Z 2021-12-10T17:16:48.672077Z 2021-12-10T17:16:48.645878Z 2021-12-10T17:16:48.620464Z 2021-12-10T17:16:48.610659Z 2021-12-10T17:16:48.406693Z 2021-12-10T17:16:48.229750Z 2021-12-10T17:16:48.195925Z 2021-12-10T17:16:48.093863Z 2021-12-10T17:16:47.959206Z 2021-12-10T17:16:47.948275Z 2021-12-10T17:16:47.922227Z 2021-12-10T17:16:47.916310Z 2021-12-10T17:16:47.894174Z 2021-12-10T17:16:47.881314Z 2021-12-10T17:16:47.692424Z 2021-12-10T17:16:47.597033Z 2021-12-10T17:16:47.430970Z 2021-12-10T17:16:47.380493Z 2021-12-10T17:16:47.211945Z 2021-12-10T17:16:47.133596Z 2021-12-10T17:16:46.895226Z 2021-12-10T17:16:46.793661Z 2021-12-10T17:16:46.783685Z 2021-12-10T17:16:46.694197Z 2021-12-10T17:16:46.686243Z 2021-12-10T17:16:46.685630Z 2021-12-10T17:16:46.553901Z 2021-12-10T17:16:46.513524Z 2021-12-10T17:16:46.448471Z 2021-12-10T17:16:46.430427Z 2021-12-10T17:16:46.402079Z 2021-12-10T17:16:45.917987Z 2021-12-10T17:16:45.823932Z 2021-12-10T17:16:45.181596Z 2021-12-10T17:21:21.886160Z 2021-12-10T17:21:21.590524Z 2021-12-10T17:21:21.358413Z 2021-12-10T17:21:21.211719Z 2021-12-10T17:21:21.150828Z 2021-12-10T17:21:21.020816Z 2021-12-10T17:21:20.897579Z 2021-12-10T17:21:20.852168Z 2021-12-10T17:21:20.534273Z 2021-12-10T17:21:20.258931Z 2021-12-10T17:21:20.254407Z 2021-12-10T17:21:20.252942Z 2021-12-10T17:21:20.073354Z 2021-12-10T17:21:20.013236Z 2021-12-10T17:21:19.941517Z 2021-12-10T17:21:19.639345Z 2021-12-10T17:21:19.458246Z 2021-12-10T17:21:19.318277Z 2021-12-10T17:21:19.196831Z 2021-12-10T17:21:19.155336Z 2021-12-10T17:21:18.884954Z 2021-12-10T17:21:18.721609Z 2021-12-10T17:21:18.417195Z 2021-12-10T17:21:18.281741Z 2021-12-10T17:21:18.225056Z 2021-12-10T17:21:18.007120Z 2021-12-10T17:21:17.887203Z 2021-12-10T17:21:17.820035Z 2021-12-10T17:21:17.719512Z 2021-12-10T17:21:17.529763Z 2021-12-10T17:21:17.361044Z 2021-12-10T17:21:17.272760Z 2021-12-10T17:21:17.114339Z 2021-12-10T17:21:17.018518Z 2021-12-10T17:21:16.984904Z 2021-12-10T17:21:16.951905Z 2021-12-10T17:21:16.848185Z 2021-12-10T17:21:16.798492Z 2021-12-10T17:21:16.542690Z 2021-12-10T17:21:16.334954Z 2021-12-10T17:21:16.316993Z 2021-12-10T17:21:16.229289Z 2021-12-10T17:21:15.892117Z 2021-12-10T17:21:15.810093Z 2021-12-10T17:21:15.548716Z 2021-12-10T17:21:15.484161Z 2021-12-10T17:21:15.146982Z 2021-12-10T17:21:15.118576Z 2021-12-10T17:21:14.850318Z 2021-12-10T17:21:14.672325Z 2021-12-10T17:21:14.665922Z 2021-12-10T17:21:14.381732Z 2021-12-10T17:21:14.375131Z 2021-12-10T17:21:14.198115Z 2021-12-10T17:21:14.139743Z 2021-12-10T17:21:14.084532Z 2021-12-10T17:21:14.070926Z 2021-12-10T17:21:13.755053Z 2021-12-10T17:21:13.672431Z 2021-12-10T17:21:13.460424Z 2021-12-10T17:21:13.410909Z 2021-12-10T17:21:13.332555Z 2021-12-10T17:21:13.303406Z 2021-12-10T17:21:13.175132Z 2021-12-10T17:21:13.159161Z 2021-12-10T17:21:13.080142Z 2021-12-10T17:21:13.079268Z 2021-12-10T17:21:12.837926Z 2021-12-10T17:21:12.842164Z 2021-12-10T17:21:12.773282Z 2021-12-10T17:21:12.574500Z 2021-12-10T17:21:12.531111Z 2021-12-10T17:21:12.485325Z 2021-12-10T17:21:12.417818Z 2021-12-10T17:21:12.332442Z 2021-12-10T17:21:12.295880Z 2021-12-10T17:21:12.096734Z 2021-12-10T17:21:11.885158Z 2021-12-10T17:21:11.814300Z 2021-12-10T17:21:11.562933Z 2021-12-10T17:21:11.345083Z 2021-12-10T17:21:11.324674Z 2021-12-10T17:21:11.233203Z 2021-12-10T17:21:11.205908Z 2021-12-10T17:21:11.199918Z 2021-12-10T17:21:11.048459Z 2021-12-10T17:21:11.003397Z 2021-12-10T17:21:10.851861Z 2021-12-10T17:21:10.841103Z 2021-12-10T17:21:10.696382Z 2021-12-10T17:21:10.568392Z 2021-12-10T17:21:10.451473Z 2021-12-10T17:21:10.441009Z 2021-12-10T17:21:10.344556Z 2021-12-10T17:21:10.332242Z 2021-12-10T17:21:08.882488Z 2021-12-10T17:21:08.871357Z 2021-12-10T17:21:08.475718Z 2021-12-10T17:21:08.356077Z 2021-12-10T17:21:07.904249Z", + "created_diff": 16.1846636, + "ended": "2021-12-10T17:22:50.765161Z", + "error_job_ids": " []", + "event_first": "2021-12-10T17:05:07.857531Z 2021-12-10T17:05:18.894475Z 2021-12-10T17:04:59.228434Z 2021-12-10T17:05:09.605165Z 2021-12-10T17:04:57.978536Z 2021-12-10T17:05:17.070002Z 2021-12-10T17:04:50.461570Z 2021-12-10T17:05:13.544591Z 2021-12-10T17:04:50.971657Z 2021-12-10T17:05:04.335316Z 2021-12-10T17:04:37.918401Z 2021-12-10T17:05:12.426847Z 2021-12-10T17:04:43.243139Z 2021-12-10T17:04:34.272714Z 2021-12-10T17:04:17.413339Z 2021-12-10T17:04:34.051053Z 2021-12-10T17:05:34.452024Z 2021-12-10T17:04:36.570506Z 2021-12-10T17:05:00.113038Z 2021-12-10T17:04:36.197485Z 2021-12-10T17:04:25.308400Z 2021-12-10T17:04:33.660548Z 2021-12-10T17:05:43.013362Z 2021-12-10T17:04:27.889761Z 2021-12-10T17:06:19.108712Z 2021-12-10T17:04:26.606836Z 2021-12-10T17:06:10.336946Z 2021-12-10T17:04:28.342065Z 2021-12-10T17:06:01.934191Z 2021-12-10T17:04:15.901421Z 2021-12-10T17:05:52.769671Z 2021-12-10T17:04:18.590841Z 2021-12-10T17:06:28.959330Z 2021-12-10T17:04:12.039656Z 2021-12-10T17:04:16.419247Z 2021-12-10T17:04:10.317160Z 2021-12-10T17:05:30.165699Z 2021-12-10T17:05:48.070582Z 2021-12-10T17:05:44.928591Z 2021-12-10T17:05:46.600219Z 2021-12-10T17:05:46.220784Z 2021-12-10T17:05:43.483766Z 2021-12-10T17:05:47.068504Z 2021-12-10T17:05:48.328493Z 2021-12-10T17:05:30.627099Z 2021-12-10T17:05:47.068504Z 2021-12-10T17:05:42.498948Z 2021-12-10T17:05:26.519497Z 2021-12-10T17:05:44.845233Z 2021-12-10T17:05:39.835320Z 2021-12-10T17:05:30.380353Z 2021-12-10T17:05:36.638926Z 2021-12-10T17:05:25.451448Z 2021-12-10T17:05:41.019481Z 2021-12-10T17:05:30.560600Z 2021-12-10T17:05:43.265679Z 2021-12-10T17:05:27.079435Z 2021-12-10T17:05:41.453988Z 2021-12-10T17:05:27.273487Z 2021-12-10T17:05:44.455514Z 2021-12-10T17:05:22.427670Z 2021-12-10T17:05:31.032481Z 2021-12-10T17:05:27.544472Z 2021-12-10T17:05:40.218605Z 2021-12-10T17:05:23.286550Z 2021-12-10T17:05:40.759632Z 2021-12-10T17:05:28.420724Z 2021-12-10T17:05:34.253828Z 2021-12-10T17:05:27.273487Z 2021-12-10T17:05:29.973350Z 2021-12-10T17:05:23.966858Z 2021-12-10T17:05:39.918956Z 2021-12-10T17:05:22.575818Z 2021-12-10T17:05:38.038699Z 2021-12-10T17:05:23.709457Z 2021-12-10T17:05:37.061394Z 2021-12-10T17:05:23.853289Z 2021-12-10T17:05:30.968501Z 2021-12-10T17:05:18.942436Z 2021-12-10T17:05:27.692735Z 2021-12-10T17:05:18.010353Z 2021-12-10T17:05:35.077173Z 2021-12-10T17:05:16.756958Z 2021-12-10T17:05:27.528596Z 2021-12-10T17:05:12.657603Z 2021-12-10T17:05:26.577487Z 2021-12-10T17:05:16.609445Z 2021-12-10T17:05:24.564778Z 2021-12-10T17:05:15.195692Z 2021-12-10T17:05:31.266965Z 2021-12-10T17:05:12.217550Z 2021-12-10T17:05:26.577487Z 2021-12-10T17:05:15.446185Z 2021-12-10T17:05:28.759932Z 2021-12-10T17:05:10.438582Z 2021-12-10T17:05:27.755746Z 2021-12-10T17:05:08.482661Z 2021-12-10T17:05:22.807774Z 2021-12-10T17:05:04.914468Z 2021-12-10T17:05:17.927814Z 2021-12-10T17:09:56.142364Z 2021-12-10T17:10:01.650592Z 2021-12-10T17:09:55.849763Z 2021-12-10T17:09:59.391588Z 2021-12-10T17:09:55.845557Z 2021-12-10T17:09:59.057537Z 2021-12-10T17:09:51.914465Z 2021-12-10T17:09:58.041724Z 2021-12-10T17:09:56.760123Z 2021-12-10T17:09:49.749898Z 2021-12-10T17:09:51.017665Z 2021-12-10T17:09:58.041724Z 2021-12-10T17:09:56.384945Z 2021-12-10T17:09:59.570225Z 2021-12-10T17:09:53.685001Z 2021-12-10T17:09:56.806451Z 2021-12-10T17:09:55.547545Z 2021-12-10T17:09:49.532621Z 2021-12-10T17:09:55.825649Z 2021-12-10T17:09:58.041724Z 2021-12-10T17:09:51.915444Z 2021-12-10T17:09:52.011695Z 2021-12-10T17:09:46.806293Z 2021-12-10T17:09:55.920809Z 2021-12-10T17:09:50.326256Z 2021-12-10T17:09:50.961068Z 2021-12-10T17:09:56.142364Z 2021-12-10T17:09:50.962013Z 2021-12-10T17:09:51.914465Z 2021-12-10T17:09:48.771541Z 2021-12-10T17:09:50.180522Z 2021-12-10T17:09:51.210915Z 2021-12-10T17:09:49.089541Z 2021-12-10T17:09:57.286710Z 2021-12-10T17:09:41.321422Z 2021-12-10T17:09:52.269173Z 2021-12-10T17:09:50.907908Z 2021-12-10T17:09:47.996436Z 2021-12-10T17:09:45.924208Z 2021-12-10T17:09:51.440557Z 2021-12-10T17:09:43.523616Z 2021-12-10T17:09:50.453484Z 2021-12-10T17:09:41.152497Z 2021-12-10T17:09:49.749898Z 2021-12-10T17:09:41.322628Z 2021-12-10T17:09:48.931655Z 2021-12-10T17:09:44.650543Z 2021-12-10T17:09:41.561009Z 2021-12-10T17:09:42.860475Z 2021-12-10T17:09:45.651069Z 2021-12-10T17:09:40.300534Z 2021-12-10T17:09:45.387602Z 2021-12-10T17:09:43.259237Z 2021-12-10T17:09:43.373560Z 2021-12-10T17:09:39.609394Z 2021-12-10T17:09:44.380637Z 2021-12-10T17:09:40.554562Z 2021-12-10T17:09:40.009007Z 2021-12-10T17:09:31.528597Z 2021-12-10T17:09:32.952976Z 2021-12-10T17:09:31.027104Z 2021-12-10T17:09:39.505583Z 2021-12-10T17:09:38.536566Z 2021-12-10T17:09:36.498591Z 2021-12-10T17:09:33.804595Z 2021-12-10T17:09:39.758151Z 2021-12-10T17:09:29.940508Z 2021-12-10T17:09:29.811520Z 2021-12-10T17:09:32.564755Z 2021-12-10T17:09:37.518653Z 2021-12-10T17:09:26.595827Z 2021-12-10T17:09:34.963581Z 2021-12-10T17:09:18.588630Z 2021-12-10T17:09:20.647830Z 2021-12-10T17:09:14.781879Z 2021-12-10T17:09:21.424731Z 2021-12-10T17:09:15.558772Z 2021-12-10T17:09:24.031631Z 2021-12-10T17:09:03.344541Z 2021-12-10T17:08:51.047528Z 2021-12-10T17:09:02.831587Z 2021-12-10T17:08:54.006181Z 2021-12-10T17:09:01.164563Z 2021-12-10T17:08:52.143926Z 2021-12-10T17:08:49.732009Z 2021-12-10T17:08:42.673417Z 2021-12-10T17:08:48.723552Z 2021-12-10T17:08:46.638587Z 2021-12-10T17:08:44.957441Z 2021-12-10T17:08:49.756654Z 2021-12-10T17:08:41.717576Z 2021-12-10T17:08:47.999009Z 2021-12-10T17:08:40.013220Z 2021-12-10T17:08:36.859894Z 2021-12-10T17:08:32.635436Z 2021-12-10T17:08:37.596671Z 2021-12-10T17:08:29.215570Z 2021-12-10T17:08:30.670605Z 2021-12-10T17:08:28.174803Z 2021-12-10T17:08:28.978379Z 2021-12-10T17:14:08.640548Z 2021-12-10T17:14:03.521167Z 2021-12-10T17:14:02.187953Z 2021-12-10T17:13:59.178661Z 2021-12-10T17:13:57.943631Z 2021-12-10T17:14:09.974421Z 2021-12-10T17:14:05.983693Z 2021-12-10T17:14:03.968104Z 2021-12-10T17:14:01.456172Z 2021-12-10T17:14:03.576238Z 2021-12-10T17:13:54.961279Z 2021-12-10T17:14:08.837457Z 2021-12-10T17:13:57.795218Z 2021-12-10T17:13:56.287795Z 2021-12-10T17:14:03.210466Z 2021-12-10T17:14:07.159353Z 2021-12-10T17:13:58.195852Z 2021-12-10T17:14:01.625592Z 2021-12-10T17:13:59.467294Z 2021-12-10T17:14:02.684369Z 2021-12-10T17:13:58.808048Z 2021-12-10T17:14:01.491775Z 2021-12-10T17:13:58.705595Z 2021-12-10T17:14:04.955230Z 2021-12-10T17:14:02.059870Z 2021-12-10T17:14:01.386333Z 2021-12-10T17:13:58.906792Z 2021-12-10T17:14:03.327378Z 2021-12-10T17:13:56.224697Z 2021-12-10T17:14:06.762706Z 2021-12-10T17:13:50.090506Z 2021-12-10T17:14:04.354521Z 2021-12-10T17:13:53.030534Z 2021-12-10T17:13:55.784234Z 2021-12-10T17:13:51.530979Z 2021-12-10T17:13:59.032813Z 2021-12-10T17:13:50.872585Z 2021-12-10T17:14:01.874941Z 2021-12-10T17:13:56.728441Z 2021-12-10T17:14:01.129136Z 2021-12-10T17:13:53.505270Z 2021-12-10T17:13:59.878520Z 2021-12-10T17:13:54.300434Z 2021-12-10T17:13:56.035670Z 2021-12-10T17:13:56.980587Z 2021-12-10T17:13:49.088138Z 2021-12-10T17:13:49.379610Z 2021-12-10T17:13:48.085654Z 2021-12-10T17:13:48.336574Z 2021-12-10T17:13:41.699981Z 2021-12-10T17:13:52.017862Z 2021-12-10T17:13:58.558537Z 2021-12-10T17:13:49.838559Z 2021-12-10T17:13:56.034632Z 2021-12-10T17:13:46.861115Z 2021-12-10T17:13:57.931623Z 2021-12-10T17:13:43.100538Z 2021-12-10T17:13:51.453473Z 2021-12-10T17:13:44.404318Z 2021-12-10T17:13:52.566065Z 2021-12-10T17:13:40.947919Z 2021-12-10T17:13:43.217326Z 2021-12-10T17:13:43.811099Z 2021-12-10T17:13:44.988955Z 2021-12-10T17:13:38.721886Z 2021-12-10T17:13:51.037684Z 2021-12-10T17:13:34.787845Z 2021-12-10T17:13:49.767714Z 2021-12-10T17:13:28.943939Z 2021-12-10T17:13:41.259669Z 2021-12-10T17:13:30.801699Z 2021-12-10T17:13:41.443171Z 2021-12-10T17:13:29.763735Z 2021-12-10T17:13:34.550607Z 2021-12-10T17:13:16.625451Z 2021-12-10T17:13:32.021630Z 2021-12-10T17:13:13.963580Z 2021-12-10T17:13:32.757505Z 2021-12-10T17:13:12.267861Z 2021-12-10T17:13:31.875935Z 2021-12-10T17:13:02.988949Z 2021-12-10T17:13:21.444785Z 2021-12-10T17:12:51.837972Z 2021-12-10T17:12:54.174847Z 2021-12-10T17:12:51.247873Z 2021-12-10T17:12:53.151538Z 2021-12-10T17:12:49.551281Z 2021-12-10T17:12:53.418652Z 2021-12-10T17:12:50.982650Z 2021-12-10T17:12:52.641856Z 2021-12-10T17:12:49.990433Z 2021-12-10T17:12:48.783479Z 2021-12-10T17:12:47.428938Z 2021-12-10T17:12:49.859526Z 2021-12-10T17:12:44.896540Z 2021-12-10T17:12:45.783138Z 2021-12-10T17:12:42.614002Z 2021-12-10T17:12:41.377263Z 2021-12-10T17:12:41.515427Z 2021-12-10T17:12:38.741713Z 2021-12-10T17:18:26.284760Z 2021-12-10T17:18:21.310816Z 2021-12-10T17:18:17.850643Z 2021-12-10T17:18:21.642772Z 2021-12-10T17:18:19.013489Z 2021-12-10T17:18:22.828329Z 2021-12-10T17:18:16.599575Z 2021-12-10T17:18:21.029715Z 2021-12-10T17:18:22.435006Z 2021-12-10T17:18:21.286562Z 2021-12-10T17:18:18.303208Z 2021-12-10T17:18:25.017667Z 2021-12-10T17:18:20.084832Z 2021-12-10T17:18:22.923123Z 2021-12-10T17:18:22.279026Z 2021-12-10T17:18:24.657611Z 2021-12-10T17:18:13.610627Z 2021-12-10T17:18:18.713420Z 2021-12-10T17:18:19.330036Z 2021-12-10T17:18:19.856916Z 2021-12-10T17:18:20.921485Z 2021-12-10T17:18:22.577800Z 2021-12-10T17:18:19.013489Z 2021-12-10T17:18:21.544580Z 2021-12-10T17:18:16.646459Z 2021-12-10T17:18:19.275334Z 2021-12-10T17:18:17.301462Z 2021-12-10T17:18:22.128476Z 2021-12-10T17:18:10.450522Z 2021-12-10T17:18:16.563430Z 2021-12-10T17:18:12.374473Z 2021-12-10T17:18:16.214957Z 2021-12-10T17:18:11.704501Z 2021-12-10T17:18:18.713420Z 2021-12-10T17:18:17.302454Z 2021-12-10T17:18:09.787478Z 2021-12-10T17:18:15.061642Z 2021-12-10T17:18:14.756536Z 2021-12-10T17:18:15.562672Z 2021-12-10T17:18:07.253086Z 2021-12-10T17:18:13.262849Z 2021-12-10T17:18:15.386378Z 2021-12-10T17:18:11.884241Z 2021-12-10T17:18:16.109420Z 2021-12-10T17:18:16.646459Z 2021-12-10T17:18:01.991577Z 2021-12-10T17:18:10.227006Z 2021-12-10T17:18:12.598432Z 2021-12-10T17:18:07.850576Z 2021-12-10T17:18:00.224610Z 2021-12-10T17:18:07.325801Z 2021-12-10T17:18:04.315186Z 2021-12-10T17:18:02.082074Z 2021-12-10T17:18:02.650497Z 2021-12-10T17:18:11.568554Z 2021-12-10T17:18:07.847483Z 2021-12-10T17:17:57.704037Z 2021-12-10T17:18:06.407652Z 2021-12-10T17:18:06.045840Z 2021-12-10T17:18:08.112975Z 2021-12-10T17:18:01.662825Z 2021-12-10T17:18:08.112975Z 2021-12-10T17:17:57.840792Z 2021-12-10T17:18:05.934805Z 2021-12-10T17:18:01.839664Z 2021-12-10T17:18:00.925559Z 2021-12-10T17:17:58.341634Z 2021-12-10T17:17:58.874843Z 2021-12-10T17:17:53.687632Z 2021-12-10T17:17:52.433479Z 2021-12-10T17:17:51.412584Z 2021-12-10T17:17:56.124148Z 2021-12-10T17:17:49.390063Z 2021-12-10T17:17:52.880537Z 2021-12-10T17:17:51.685609Z 2021-12-10T17:17:43.774800Z 2021-12-10T17:17:43.734690Z 2021-12-10T17:17:47.300600Z 2021-12-10T17:17:47.203018Z 2021-12-10T17:17:43.696701Z 2021-12-10T17:17:43.945591Z 2021-12-10T17:17:44.575677Z 2021-12-10T17:17:41.530655Z 2021-12-10T17:17:34.004543Z 2021-12-10T17:17:12.654108Z 2021-12-10T17:17:08.714450Z 2021-12-10T17:17:12.332005Z 2021-12-10T17:17:06.358578Z 2021-12-10T17:17:09.868334Z 2021-12-10T17:17:06.358578Z 2021-12-10T17:17:09.069619Z 2021-12-10T17:17:06.002376Z 2021-12-10T17:17:07.291820Z 2021-12-10T17:17:04.917707Z 2021-12-10T17:17:05.242119Z 2021-12-10T17:17:03.331796Z 2021-12-10T17:17:05.687551Z 2021-12-10T17:16:57.052579Z 2021-12-10T17:16:55.506182Z 2021-12-10T17:16:52.183610Z 2021-12-10T17:22:42.433428Z 2021-12-10T17:22:46.998506Z 2021-12-10T17:22:47.617671Z 2021-12-10T17:22:45.966499Z 2021-12-10T17:22:47.860167Z 2021-12-10T17:22:44.935045Z 2021-12-10T17:22:46.595768Z 2021-12-10T17:22:46.598956Z 2021-12-10T17:22:44.636330Z 2021-12-10T17:22:47.216553Z 2021-12-10T17:22:41.218555Z 2021-12-10T17:22:40.724806Z 2021-12-10T17:22:36.559648Z 2021-12-10T17:22:43.979708Z 2021-12-10T17:22:42.796141Z 2021-12-10T17:22:43.420049Z 2021-12-10T17:22:42.434478Z 2021-12-10T17:22:46.029781Z 2021-12-10T17:22:40.686441Z 2021-12-10T17:22:44.460683Z 2021-12-10T17:22:40.964436Z 2021-12-10T17:22:39.581078Z 2021-12-10T17:22:39.044606Z 2021-12-10T17:22:42.691268Z 2021-12-10T17:22:42.684430Z 2021-12-10T17:22:42.911834Z 2021-12-10T17:22:35.746440Z 2021-12-10T17:22:39.581078Z 2021-12-10T17:22:38.824956Z 2021-12-10T17:22:43.979708Z 2021-12-10T17:22:40.088096Z 2021-12-10T17:22:37.092664Z 2021-12-10T17:22:34.837456Z 2021-12-10T17:22:34.529623Z 2021-12-10T17:22:42.686617Z 2021-12-10T17:22:37.070577Z 2021-12-10T17:22:39.696440Z 2021-12-10T17:22:43.953422Z 2021-12-10T17:22:39.728774Z 2021-12-10T17:22:38.837868Z 2021-12-10T17:22:39.696440Z 2021-12-10T17:22:40.341504Z 2021-12-10T17:22:38.649850Z 2021-12-10T17:22:38.562533Z 2021-12-10T17:22:40.428857Z 2021-12-10T17:22:38.336883Z 2021-12-10T17:22:36.736130Z 2021-12-10T17:22:37.579329Z 2021-12-10T17:22:31.969736Z 2021-12-10T17:22:28.588537Z 2021-12-10T17:22:36.789926Z 2021-12-10T17:22:26.155605Z 2021-12-10T17:22:24.439960Z 2021-12-10T17:22:17.068925Z 2021-12-10T17:22:26.890594Z 2021-12-10T17:22:28.417925Z 2021-12-10T17:22:24.918592Z 2021-12-10T17:22:15.727100Z 2021-12-10T17:22:17.449785Z 2021-12-10T17:22:17.533613Z 2021-12-10T17:22:19.416676Z 2021-12-10T17:22:21.562740Z 2021-12-10T17:22:12.082565Z 2021-12-10T17:22:17.822321Z 2021-12-10T17:22:18.907159Z 2021-12-10T17:22:14.071754Z 2021-12-10T17:22:19.175642Z 2021-12-10T17:22:08.894367Z 2021-12-10T17:22:12.288456Z 2021-12-10T17:22:24.802684Z 2021-12-10T17:22:13.249697Z 2021-12-10T17:22:07.780469Z 2021-12-10T17:22:07.470884Z 2021-12-10T17:22:10.502485Z 2021-12-10T17:22:04.545498Z 2021-12-10T17:22:15.825601Z 2021-12-10T17:21:59.693141Z 2021-12-10T17:22:01.769984Z 2021-12-10T17:21:53.163043Z 2021-12-10T17:22:06.814674Z 2021-12-10T17:22:06.485919Z 2021-12-10T17:22:02.027253Z 2021-12-10T17:21:50.671278Z 2021-12-10T17:21:56.745427Z 2021-12-10T17:21:40.722568Z 2021-12-10T17:21:43.993989Z 2021-12-10T17:21:44.098746Z 2021-12-10T17:21:51.297196Z 2021-12-10T17:21:44.443915Z 2021-12-10T17:21:41.580303Z 2021-12-10T17:21:24.840051Z 2021-12-10T17:21:27.618589Z 2021-12-10T17:21:25.637650Z 2021-12-10T17:21:26.062369Z 2021-12-10T17:21:24.683070Z 2021-12-10T17:21:20.738538Z 2021-12-10T17:21:18.081421Z 2021-12-10T17:21:19.037684Z 2021-12-10T17:21:16.456829Z 2021-12-10T17:21:17.297580Z", + "event_last": "2021-12-10T17:05:10.298534Z 2021-12-10T17:05:23.180351Z 2021-12-10T17:05:02.153258Z 2021-12-10T17:05:13.262340Z 2021-12-10T17:05:00.021274Z 2021-12-10T17:05:21.675753Z 2021-12-10T17:04:54.950338Z 2021-12-10T17:05:17.693383Z 2021-12-10T17:04:54.619018Z 2021-12-10T17:05:08.925397Z 2021-12-10T17:04:41.379574Z 2021-12-10T17:05:16.262381Z 2021-12-10T17:04:46.885275Z 2021-12-10T17:04:37.217008Z 2021-12-10T17:04:17.826446Z 2021-12-10T17:04:37.338360Z 2021-12-10T17:05:35.250146Z 2021-12-10T17:04:40.257479Z 2021-12-10T17:05:03.412272Z 2021-12-10T17:04:40.098509Z 2021-12-10T17:04:28.194087Z 2021-12-10T17:04:36.649917Z 2021-12-10T17:05:43.670826Z 2021-12-10T17:04:30.557443Z 2021-12-10T17:06:19.662431Z 2021-12-10T17:04:28.332633Z 2021-12-10T17:06:10.766611Z 2021-12-10T17:04:30.907396Z 2021-12-10T17:06:02.333468Z 2021-12-10T17:04:17.012850Z 2021-12-10T17:05:53.635019Z 2021-12-10T17:04:20.095641Z 2021-12-10T17:06:30.780269Z 2021-12-10T17:04:13.330612Z 2021-12-10T17:04:17.416750Z 2021-12-10T17:04:11.554366Z 2021-12-10T17:05:31.730034Z 2021-12-10T17:05:49.379980Z 2021-12-10T17:05:47.431720Z 2021-12-10T17:05:48.417388Z 2021-12-10T17:05:48.326534Z 2021-12-10T17:05:46.273911Z 2021-12-10T17:05:48.716495Z 2021-12-10T17:05:51.072280Z 2021-12-10T17:05:32.144282Z 2021-12-10T17:05:48.817926Z 2021-12-10T17:05:44.845233Z 2021-12-10T17:05:28.436035Z 2021-12-10T17:05:46.318796Z 2021-12-10T17:05:42.601625Z 2021-12-10T17:05:31.920832Z 2021-12-10T17:05:40.338445Z 2021-12-10T17:05:28.601280Z 2021-12-10T17:05:43.487296Z 2021-12-10T17:05:31.915108Z 2021-12-10T17:05:45.793323Z 2021-12-10T17:05:29.152115Z 2021-12-10T17:05:44.750498Z 2021-12-10T17:05:29.543483Z 2021-12-10T17:05:46.563362Z 2021-12-10T17:05:25.810374Z 2021-12-10T17:05:34.898568Z 2021-12-10T17:05:29.823399Z 2021-12-10T17:05:43.449513Z 2021-12-10T17:05:26.474929Z 2021-12-10T17:05:43.727215Z 2021-12-10T17:05:30.195824Z 2021-12-10T17:05:37.837391Z 2021-12-10T17:05:29.543483Z 2021-12-10T17:05:34.691434Z 2021-12-10T17:05:27.339366Z 2021-12-10T17:05:42.464917Z 2021-12-10T17:05:24.918619Z 2021-12-10T17:05:41.530395Z 2021-12-10T17:05:26.477682Z 2021-12-10T17:05:40.839689Z 2021-12-10T17:05:27.451459Z 2021-12-10T17:05:36.383547Z 2021-12-10T17:05:22.329274Z 2021-12-10T17:05:32.585457Z 2021-12-10T17:05:21.031276Z 2021-12-10T17:05:39.099710Z 2021-12-10T17:05:20.074286Z 2021-12-10T17:05:32.058879Z 2021-12-10T17:05:15.742597Z 2021-12-10T17:05:31.076412Z 2021-12-10T17:05:19.468474Z 2021-12-10T17:05:28.352411Z 2021-12-10T17:05:17.613269Z 2021-12-10T17:05:35.064229Z 2021-12-10T17:05:15.022283Z 2021-12-10T17:05:32.164359Z 2021-12-10T17:05:18.536267Z 2021-12-10T17:05:33.877048Z 2021-12-10T17:05:13.595507Z 2021-12-10T17:05:32.403471Z 2021-12-10T17:05:11.821603Z 2021-12-10T17:05:26.923343Z 2021-12-10T17:05:08.286036Z 2021-12-10T17:05:22.541711Z 2021-12-10T17:09:57.487279Z 2021-12-10T17:10:02.086803Z 2021-12-10T17:09:57.311298Z 2021-12-10T17:10:01.318805Z 2021-12-10T17:09:57.550123Z 2021-12-10T17:10:00.723053Z 2021-12-10T17:09:54.367908Z 2021-12-10T17:09:59.907860Z 2021-12-10T17:09:58.054092Z 2021-12-10T17:09:53.110444Z 2021-12-10T17:09:53.850502Z 2021-12-10T17:10:00.165939Z 2021-12-10T17:09:57.807327Z 2021-12-10T17:10:01.200941Z 2021-12-10T17:09:56.218379Z 2021-12-10T17:09:58.506365Z 2021-12-10T17:09:57.319455Z 2021-12-10T17:09:53.086406Z 2021-12-10T17:09:58.268613Z 2021-12-10T17:09:59.567316Z 2021-12-10T17:09:54.561277Z 2021-12-10T17:09:55.587314Z 2021-12-10T17:09:49.806325Z 2021-12-10T17:09:58.117342Z 2021-12-10T17:09:52.958632Z 2021-12-10T17:09:54.133416Z 2021-12-10T17:09:57.699884Z 2021-12-10T17:09:55.445828Z 2021-12-10T17:09:54.539496Z 2021-12-10T17:09:52.404546Z 2021-12-10T17:09:54.158757Z 2021-12-10T17:09:55.796358Z 2021-12-10T17:09:52.104045Z 2021-12-10T17:09:58.650526Z 2021-12-10T17:09:45.987288Z 2021-12-10T17:09:55.556131Z 2021-12-10T17:09:54.137025Z 2021-12-10T17:09:51.752797Z 2021-12-10T17:09:50.120520Z 2021-12-10T17:09:55.428447Z 2021-12-10T17:09:46.581249Z 2021-12-10T17:09:54.422337Z 2021-12-10T17:09:45.150891Z 2021-12-10T17:09:54.034516Z 2021-12-10T17:09:45.124934Z 2021-12-10T17:09:53.179231Z 2021-12-10T17:09:48.085289Z 2021-12-10T17:09:44.895960Z 2021-12-10T17:09:47.021899Z 2021-12-10T17:09:49.329490Z 2021-12-10T17:09:44.162299Z 2021-12-10T17:09:49.761494Z 2021-12-10T17:09:47.371522Z 2021-12-10T17:09:47.464402Z 2021-12-10T17:09:44.190678Z 2021-12-10T17:09:48.742352Z 2021-12-10T17:09:43.826779Z 2021-12-10T17:09:44.252805Z 2021-12-10T17:09:34.903458Z 2021-12-10T17:09:37.290492Z 2021-12-10T17:09:34.488272Z 2021-12-10T17:09:43.643314Z 2021-12-10T17:09:42.684605Z 2021-12-10T17:09:41.149008Z 2021-12-10T17:09:37.093377Z 2021-12-10T17:09:43.373560Z 2021-12-10T17:09:32.815720Z 2021-12-10T17:09:33.698011Z 2021-12-10T17:09:36.397673Z 2021-12-10T17:09:41.107356Z 2021-12-10T17:09:30.203300Z 2021-12-10T17:09:38.347482Z 2021-12-10T17:09:21.983726Z 2021-12-10T17:09:25.145308Z 2021-12-10T17:09:18.275276Z 2021-12-10T17:09:26.005854Z 2021-12-10T17:09:20.512321Z 2021-12-10T17:09:26.762291Z 2021-12-10T17:09:07.087265Z 2021-12-10T17:08:54.507839Z 2021-12-10T17:09:07.258347Z 2021-12-10T17:08:57.758337Z 2021-12-10T17:09:04.374343Z 2021-12-10T17:08:55.892867Z 2021-12-10T17:08:52.490260Z 2021-12-10T17:08:44.209413Z 2021-12-10T17:08:51.364012Z 2021-12-10T17:08:49.376722Z 2021-12-10T17:08:46.825280Z 2021-12-10T17:08:51.982369Z 2021-12-10T17:08:43.528318Z 2021-12-10T17:08:50.718849Z 2021-12-10T17:08:41.250574Z 2021-12-10T17:08:38.129321Z 2021-12-10T17:08:34.939768Z 2021-12-10T17:08:38.614510Z 2021-12-10T17:08:30.393898Z 2021-12-10T17:08:31.979251Z 2021-12-10T17:08:29.560828Z 2021-12-10T17:08:30.340190Z 2021-12-10T17:14:10.060355Z 2021-12-10T17:14:04.793780Z 2021-12-10T17:14:05.711383Z 2021-12-10T17:14:02.515601Z 2021-12-10T17:14:01.872252Z 2021-12-10T17:14:10.171347Z 2021-12-10T17:14:08.481309Z 2021-12-10T17:14:04.790692Z 2021-12-10T17:14:03.382164Z 2021-12-10T17:14:06.432513Z 2021-12-10T17:13:58.055635Z 2021-12-10T17:14:09.974421Z 2021-12-10T17:14:01.304475Z 2021-12-10T17:13:59.826362Z 2021-12-10T17:14:04.578908Z 2021-12-10T17:14:08.992982Z 2021-12-10T17:14:00.784149Z 2021-12-10T17:14:05.578591Z 2021-12-10T17:14:02.831652Z 2021-12-10T17:14:05.577281Z 2021-12-10T17:14:01.895305Z 2021-12-10T17:14:04.534459Z 2021-12-10T17:14:01.997620Z 2021-12-10T17:14:07.784659Z 2021-12-10T17:14:04.310998Z 2021-12-10T17:14:05.907359Z 2021-12-10T17:14:02.242969Z 2021-12-10T17:14:05.561817Z 2021-12-10T17:13:58.627667Z 2021-12-10T17:14:08.617847Z 2021-12-10T17:13:54.192358Z 2021-12-10T17:14:06.458779Z 2021-12-10T17:13:56.166915Z 2021-12-10T17:13:59.548143Z 2021-12-10T17:13:55.455290Z 2021-12-10T17:14:02.065640Z 2021-12-10T17:13:54.972417Z 2021-12-10T17:14:04.846322Z 2021-12-10T17:14:00.301231Z 2021-12-10T17:14:05.977397Z 2021-12-10T17:13:57.328790Z 2021-12-10T17:14:04.703846Z 2021-12-10T17:13:58.244600Z 2021-12-10T17:13:58.329369Z 2021-12-10T17:14:00.683200Z 2021-12-10T17:13:53.117333Z 2021-12-10T17:13:52.311126Z 2021-12-10T17:13:52.414347Z 2021-12-10T17:13:51.439973Z 2021-12-10T17:13:46.074537Z 2021-12-10T17:13:55.743402Z 2021-12-10T17:14:02.652365Z 2021-12-10T17:13:54.651782Z 2021-12-10T17:13:58.949299Z 2021-12-10T17:13:49.652303Z 2021-12-10T17:14:02.328417Z 2021-12-10T17:13:46.464269Z 2021-12-10T17:13:56.114278Z 2021-12-10T17:13:47.245966Z 2021-12-10T17:13:55.784710Z 2021-12-10T17:13:44.785251Z 2021-12-10T17:13:46.956647Z 2021-12-10T17:13:47.380272Z 2021-12-10T17:13:48.517807Z 2021-12-10T17:13:42.313313Z 2021-12-10T17:13:55.768383Z 2021-12-10T17:13:37.344273Z 2021-12-10T17:13:53.544362Z 2021-12-10T17:13:31.790344Z 2021-12-10T17:13:44.148355Z 2021-12-10T17:13:34.041651Z 2021-12-10T17:13:46.074537Z 2021-12-10T17:13:35.060603Z 2021-12-10T17:13:37.458592Z 2021-12-10T17:13:21.253262Z 2021-12-10T17:13:36.128865Z 2021-12-10T17:13:17.239255Z 2021-12-10T17:13:36.119342Z 2021-12-10T17:13:16.903362Z 2021-12-10T17:13:36.550305Z 2021-12-10T17:13:05.702210Z 2021-12-10T17:13:25.523486Z 2021-12-10T17:12:54.845052Z 2021-12-10T17:12:55.794216Z 2021-12-10T17:12:51.872308Z 2021-12-10T17:12:55.020444Z 2021-12-10T17:12:50.317990Z 2021-12-10T17:12:54.925918Z 2021-12-10T17:12:51.793234Z 2021-12-10T17:12:53.978902Z 2021-12-10T17:12:50.813462Z 2021-12-10T17:12:49.859526Z 2021-12-10T17:12:49.560862Z 2021-12-10T17:12:52.641856Z 2021-12-10T17:12:45.899274Z 2021-12-10T17:12:46.550276Z 2021-12-10T17:12:43.391263Z 2021-12-10T17:12:42.813547Z 2021-12-10T17:12:42.617119Z 2021-12-10T17:12:40.351261Z 2021-12-10T17:18:27.002239Z 2021-12-10T17:18:24.196367Z 2021-12-10T17:18:21.640324Z 2021-12-10T17:18:23.916435Z 2021-12-10T17:18:21.478722Z 2021-12-10T17:18:24.978328Z 2021-12-10T17:18:19.440171Z 2021-12-10T17:18:23.416367Z 2021-12-10T17:18:23.478485Z 2021-12-10T17:18:23.655338Z 2021-12-10T17:18:20.783995Z 2021-12-10T17:18:26.562586Z 2021-12-10T17:18:22.257622Z 2021-12-10T17:18:25.448530Z 2021-12-10T17:18:23.447076Z 2021-12-10T17:18:26.247079Z 2021-12-10T17:18:16.435363Z 2021-12-10T17:18:21.405818Z 2021-12-10T17:18:22.368626Z 2021-12-10T17:18:22.258635Z 2021-12-10T17:18:22.796100Z 2021-12-10T17:18:24.875742Z 2021-12-10T17:18:22.200601Z 2021-12-10T17:18:23.873595Z 2021-12-10T17:18:19.993236Z 2021-12-10T17:18:22.785262Z 2021-12-10T17:18:21.107161Z 2021-12-10T17:18:25.000930Z 2021-12-10T17:18:13.696831Z 2021-12-10T17:18:19.853606Z 2021-12-10T17:18:14.908330Z 2021-12-10T17:18:19.623288Z 2021-12-10T17:18:15.670982Z 2021-12-10T17:18:21.478071Z 2021-12-10T17:18:20.482010Z 2021-12-10T17:18:13.645321Z 2021-12-10T17:18:17.774958Z 2021-12-10T17:18:17.544022Z 2021-12-10T17:18:18.166207Z 2021-12-10T17:18:11.988200Z 2021-12-10T17:18:16.608826Z 2021-12-10T17:18:18.919488Z 2021-12-10T17:18:15.183320Z 2021-12-10T17:18:19.742589Z 2021-12-10T17:18:20.536162Z 2021-12-10T17:18:06.475348Z 2021-12-10T17:18:13.343302Z 2021-12-10T17:18:16.261269Z 2021-12-10T17:18:11.690379Z 2021-12-10T17:18:04.027283Z 2021-12-10T17:18:10.929134Z 2021-12-10T17:18:07.847483Z 2021-12-10T17:18:07.101489Z 2021-12-10T17:18:04.641294Z 2021-12-10T17:18:15.349284Z 2021-12-10T17:18:11.803318Z 2021-12-10T17:18:00.220404Z 2021-12-10T17:18:10.717551Z 2021-12-10T17:18:10.227006Z 2021-12-10T17:18:12.319283Z 2021-12-10T17:18:04.990532Z 2021-12-10T17:18:12.233999Z 2021-12-10T17:18:01.929355Z 2021-12-10T17:18:10.330032Z 2021-12-10T17:18:05.788642Z 2021-12-10T17:18:04.895309Z 2021-12-10T17:18:03.054416Z 2021-12-10T17:18:01.536803Z 2021-12-10T17:17:57.463477Z 2021-12-10T17:17:56.954754Z 2021-12-10T17:17:56.312165Z 2021-12-10T17:17:59.778022Z 2021-12-10T17:17:52.964436Z 2021-12-10T17:17:57.177286Z 2021-12-10T17:17:56.083018Z 2021-12-10T17:17:47.800304Z 2021-12-10T17:17:47.224894Z 2021-12-10T17:17:50.958354Z 2021-12-10T17:17:51.479596Z 2021-12-10T17:17:47.596396Z 2021-12-10T17:17:48.012344Z 2021-12-10T17:17:48.587346Z 2021-12-10T17:17:44.777348Z 2021-12-10T17:17:38.632406Z 2021-12-10T17:17:15.038994Z 2021-12-10T17:17:10.114676Z 2021-12-10T17:17:14.790907Z 2021-12-10T17:17:08.048562Z 2021-12-10T17:17:12.034431Z 2021-12-10T17:17:08.756403Z 2021-12-10T17:17:10.753337Z 2021-12-10T17:17:07.018834Z 2021-12-10T17:17:09.078992Z 2021-12-10T17:17:05.989370Z 2021-12-10T17:17:07.021288Z 2021-12-10T17:17:04.392086Z 2021-12-10T17:17:07.264856Z 2021-12-10T17:16:57.887765Z 2021-12-10T17:16:56.424960Z 2021-12-10T17:16:54.841561Z 2021-12-10T17:22:45.260419Z 2021-12-10T17:22:48.610514Z 2021-12-10T17:22:49.352986Z 2021-12-10T17:22:48.582476Z 2021-12-10T17:22:49.352986Z 2021-12-10T17:22:48.012177Z 2021-12-10T17:22:47.860167Z 2021-12-10T17:22:48.787578Z 2021-12-10T17:22:46.689173Z 2021-12-10T17:22:50.765161Z 2021-12-10T17:22:44.268667Z 2021-12-10T17:22:43.631382Z 2021-12-10T17:22:39.109287Z 2021-12-10T17:22:47.032433Z 2021-12-10T17:22:46.228817Z 2021-12-10T17:22:46.092809Z 2021-12-10T17:22:46.020675Z 2021-12-10T17:22:48.137469Z 2021-12-10T17:22:44.300344Z 2021-12-10T17:22:46.598956Z 2021-12-10T17:22:44.147970Z 2021-12-10T17:22:41.975494Z 2021-12-10T17:22:42.745492Z 2021-12-10T17:22:45.398337Z 2021-12-10T17:22:46.022110Z 2021-12-10T17:22:45.401685Z 2021-12-10T17:22:39.579019Z 2021-12-10T17:22:41.658764Z 2021-12-10T17:22:41.654366Z 2021-12-10T17:22:45.815094Z 2021-12-10T17:22:43.307629Z 2021-12-10T17:22:40.422307Z 2021-12-10T17:22:38.091358Z 2021-12-10T17:22:37.598407Z 2021-12-10T17:22:46.399272Z 2021-12-10T17:22:39.651172Z 2021-12-10T17:22:42.372264Z 2021-12-10T17:22:45.874887Z 2021-12-10T17:22:42.544230Z 2021-12-10T17:22:42.310921Z 2021-12-10T17:22:43.279900Z 2021-12-10T17:22:43.358667Z 2021-12-10T17:22:41.800266Z 2021-12-10T17:22:43.478549Z 2021-12-10T17:22:44.934764Z 2021-12-10T17:22:41.009771Z 2021-12-10T17:22:39.415275Z 2021-12-10T17:22:40.724806Z 2021-12-10T17:22:35.534326Z 2021-12-10T17:22:31.849338Z 2021-12-10T17:22:39.584134Z 2021-12-10T17:22:29.959769Z 2021-12-10T17:22:27.746252Z 2021-12-10T17:22:22.532341Z 2021-12-10T17:22:28.923964Z 2021-12-10T17:22:31.687330Z 2021-12-10T17:22:28.222314Z 2021-12-10T17:22:20.450307Z 2021-12-10T17:22:21.559293Z 2021-12-10T17:22:21.672347Z 2021-12-10T17:22:23.064370Z 2021-12-10T17:22:25.689295Z 2021-12-10T17:22:15.726254Z 2021-12-10T17:22:22.244945Z 2021-12-10T17:22:23.078534Z 2021-12-10T17:22:20.183325Z 2021-12-10T17:22:23.396265Z 2021-12-10T17:22:13.895674Z 2021-12-10T17:22:16.679314Z 2021-12-10T17:22:28.120707Z 2021-12-10T17:22:16.546327Z 2021-12-10T17:22:11.470375Z 2021-12-10T17:22:11.407260Z 2021-12-10T17:22:14.828478Z 2021-12-10T17:22:08.537371Z 2021-12-10T17:22:20.888682Z 2021-12-10T17:22:03.281750Z 2021-12-10T17:22:06.133229Z 2021-12-10T17:21:56.952162Z 2021-12-10T17:22:11.283358Z 2021-12-10T17:22:11.204358Z 2021-12-10T17:22:07.198206Z 2021-12-10T17:21:55.816461Z 2021-12-10T17:22:00.368409Z 2021-12-10T17:21:42.364287Z 2021-12-10T17:21:47.358848Z 2021-12-10T17:21:48.501025Z 2021-12-10T17:21:55.503316Z 2021-12-10T17:21:48.285244Z 2021-12-10T17:21:43.912893Z 2021-12-10T17:21:25.757154Z 2021-12-10T17:21:28.781832Z 2021-12-10T17:21:26.443646Z 2021-12-10T17:21:27.207599Z 2021-12-10T17:21:25.684302Z 2021-12-10T17:21:21.882448Z 2021-12-10T17:21:21.145756Z 2021-12-10T17:21:20.274507Z 2021-12-10T17:21:17.259247Z 2021-12-10T17:21:18.517442Z", + "failed_job_ids": " []", + "finished": "2021-12-10T17:05:15.381201Z 2021-12-10T17:05:28.978997Z 2021-12-10T17:05:06.517956Z 2021-12-10T17:05:19.630499Z 2021-12-10T17:05:05.206108Z 2021-12-10T17:05:28.142318Z 2021-12-10T17:05:01.292173Z 2021-12-10T17:05:24.650088Z 2021-12-10T17:04:59.896094Z 2021-12-10T17:05:14.596039Z 2021-12-10T17:04:46.576232Z 2021-12-10T17:05:22.856150Z 2021-12-10T17:04:52.372370Z 2021-12-10T17:04:42.475296Z 2021-12-10T17:04:21.555714Z 2021-12-10T17:04:41.789231Z 2021-12-10T17:05:39.511844Z 2021-12-10T17:04:46.350083Z 2021-12-10T17:05:08.607927Z 2021-12-10T17:04:44.419888Z 2021-12-10T17:04:30.464139Z 2021-12-10T17:04:41.679241Z 2021-12-10T17:05:47.369053Z 2021-12-10T17:04:34.486204Z 2021-12-10T17:06:23.434796Z 2021-12-10T17:04:32.041629Z 2021-12-10T17:06:14.687715Z 2021-12-10T17:04:35.269494Z 2021-12-10T17:06:06.268766Z 2021-12-10T17:04:21.046854Z 2021-12-10T17:05:56.859408Z 2021-12-10T17:04:24.402304Z 2021-12-10T17:06:33.526594Z 2021-12-10T17:04:17.873915Z 2021-12-10T17:04:20.805935Z 2021-12-10T17:04:15.820946Z 2021-12-10T17:05:35.328912Z 2021-12-10T17:05:53.520944Z 2021-12-10T17:05:51.739478Z 2021-12-10T17:05:53.193921Z 2021-12-10T17:05:52.622980Z 2021-12-10T17:05:50.859743Z 2021-12-10T17:05:52.478623Z 2021-12-10T17:05:53.441292Z 2021-12-10T17:05:36.093680Z 2021-12-10T17:05:53.594600Z 2021-12-10T17:05:48.946390Z 2021-12-10T17:05:33.823827Z 2021-12-10T17:05:50.441091Z 2021-12-10T17:05:48.522911Z 2021-12-10T17:05:36.529931Z 2021-12-10T17:05:46.068261Z 2021-12-10T17:05:33.208863Z 2021-12-10T17:05:48.902128Z 2021-12-10T17:05:36.390871Z 2021-12-10T17:05:50.973639Z 2021-12-10T17:05:34.417855Z 2021-12-10T17:05:49.049039Z 2021-12-10T17:05:34.579200Z 2021-12-10T17:05:51.027745Z 2021-12-10T17:05:31.489214Z 2021-12-10T17:05:40.660587Z 2021-12-10T17:05:34.765849Z 2021-12-10T17:05:49.333153Z 2021-12-10T17:05:32.438320Z 2021-12-10T17:05:48.575231Z 2021-12-10T17:05:34.659129Z 2021-12-10T17:05:43.583252Z 2021-12-10T17:05:34.571216Z 2021-12-10T17:05:39.580755Z 2021-12-10T17:05:31.612134Z 2021-12-10T17:05:46.903612Z 2021-12-10T17:05:30.549173Z 2021-12-10T17:05:46.875784Z 2021-12-10T17:05:31.605472Z 2021-12-10T17:05:46.277073Z 2021-12-10T17:05:31.608972Z 2021-12-10T17:05:42.155050Z 2021-12-10T17:05:28.176976Z 2021-12-10T17:05:39.060419Z 2021-12-10T17:05:26.211200Z 2021-12-10T17:05:44.387369Z 2021-12-10T17:05:25.428426Z 2021-12-10T17:05:38.121590Z 2021-12-10T17:05:21.197254Z 2021-12-10T17:05:38.362040Z 2021-12-10T17:05:25.524615Z 2021-12-10T17:05:34.943556Z 2021-12-10T17:05:23.035108Z 2021-12-10T17:05:41.174869Z 2021-12-10T17:05:21.095399Z 2021-12-10T17:05:36.044428Z 2021-12-10T17:05:23.740152Z 2021-12-10T17:05:39.593180Z 2021-12-10T17:05:19.003307Z 2021-12-10T17:05:38.112716Z 2021-12-10T17:05:17.876149Z 2021-12-10T17:05:32.575300Z 2021-12-10T17:05:13.264139Z 2021-12-10T17:05:28.562541Z 2021-12-10T17:10:01.287280Z 2021-12-10T17:10:05.528591Z 2021-12-10T17:10:01.550411Z 2021-12-10T17:10:05.091456Z 2021-12-10T17:10:02.172170Z 2021-12-10T17:10:04.799565Z 2021-12-10T17:09:59.573252Z 2021-12-10T17:10:04.469540Z 2021-12-10T17:10:01.939598Z 2021-12-10T17:09:57.771339Z 2021-12-10T17:09:59.187115Z 2021-12-10T17:10:05.273520Z 2021-12-10T17:10:01.921268Z 2021-12-10T17:10:05.352745Z 2021-12-10T17:10:00.540262Z 2021-12-10T17:10:02.289349Z 2021-12-10T17:10:01.590914Z 2021-12-10T17:09:58.657856Z 2021-12-10T17:10:02.293622Z 2021-12-10T17:10:04.141863Z 2021-12-10T17:09:59.120481Z 2021-12-10T17:09:59.565947Z 2021-12-10T17:09:54.807434Z 2021-12-10T17:10:02.965872Z 2021-12-10T17:09:56.889095Z 2021-12-10T17:10:00.726289Z 2021-12-10T17:10:02.187850Z 2021-12-10T17:10:00.183195Z 2021-12-10T17:10:00.070865Z 2021-12-10T17:09:57.817205Z 2021-12-10T17:10:00.001611Z 2021-12-10T17:09:59.823751Z 2021-12-10T17:09:56.246254Z 2021-12-10T17:10:03.577766Z 2021-12-10T17:09:51.344807Z 2021-12-10T17:10:00.571705Z 2021-12-10T17:09:59.216105Z 2021-12-10T17:09:56.695441Z 2021-12-10T17:09:54.170266Z 2021-12-10T17:10:00.538171Z 2021-12-10T17:09:53.108357Z 2021-12-10T17:10:00.180145Z 2021-12-10T17:09:51.012009Z 2021-12-10T17:09:59.005679Z 2021-12-10T17:09:51.365308Z 2021-12-10T17:09:59.143942Z 2021-12-10T17:09:53.274009Z 2021-12-10T17:09:50.177204Z 2021-12-10T17:09:51.971967Z 2021-12-10T17:09:54.598599Z 2021-12-10T17:09:48.527164Z 2021-12-10T17:09:56.168983Z 2021-12-10T17:09:54.665863Z 2021-12-10T17:09:53.324790Z 2021-12-10T17:09:49.018214Z 2021-12-10T17:09:54.664471Z 2021-12-10T17:09:48.932241Z 2021-12-10T17:09:50.381350Z 2021-12-10T17:09:40.804032Z 2021-12-10T17:09:43.415089Z 2021-12-10T17:09:39.349221Z 2021-12-10T17:09:49.389435Z 2021-12-10T17:09:48.293956Z 2021-12-10T17:09:45.827571Z 2021-12-10T17:09:42.100192Z 2021-12-10T17:09:50.395914Z 2021-12-10T17:09:39.009235Z 2021-12-10T17:09:39.232204Z 2021-12-10T17:09:41.519751Z 2021-12-10T17:09:46.052685Z 2021-12-10T17:09:35.270952Z 2021-12-10T17:09:43.997028Z 2021-12-10T17:09:27.243205Z 2021-12-10T17:09:30.966218Z 2021-12-10T17:09:25.792866Z 2021-12-10T17:09:29.968054Z 2021-12-10T17:09:27.062005Z 2021-12-10T17:09:32.397188Z 2021-12-10T17:09:13.743192Z 2021-12-10T17:09:00.302311Z 2021-12-10T17:09:13.303288Z 2021-12-10T17:09:02.575763Z 2021-12-10T17:09:09.787522Z 2021-12-10T17:09:00.450996Z 2021-12-10T17:08:57.385912Z 2021-12-10T17:08:48.526352Z 2021-12-10T17:08:55.780259Z 2021-12-10T17:08:55.212751Z 2021-12-10T17:08:51.616976Z 2021-12-10T17:08:58.306641Z 2021-12-10T17:08:47.487160Z 2021-12-10T17:08:55.099271Z 2021-12-10T17:08:45.486511Z 2021-12-10T17:08:42.066047Z 2021-12-10T17:08:38.462569Z 2021-12-10T17:08:43.280078Z 2021-12-10T17:08:34.500322Z 2021-12-10T17:08:35.964709Z 2021-12-10T17:08:33.909934Z 2021-12-10T17:08:34.058325Z 2021-12-10T17:14:14.085062Z 2021-12-10T17:14:08.700659Z 2021-12-10T17:14:10.551989Z 2021-12-10T17:14:07.341399Z 2021-12-10T17:14:06.563799Z 2021-12-10T17:14:14.459456Z 2021-12-10T17:14:12.773191Z 2021-12-10T17:14:08.878823Z 2021-12-10T17:14:07.415283Z 2021-12-10T17:14:11.427771Z 2021-12-10T17:14:02.843475Z 2021-12-10T17:14:14.078604Z 2021-12-10T17:14:05.444542Z 2021-12-10T17:14:05.251717Z 2021-12-10T17:14:08.152365Z 2021-12-10T17:14:13.662850Z 2021-12-10T17:14:05.684730Z 2021-12-10T17:14:11.636045Z 2021-12-10T17:14:07.148331Z 2021-12-10T17:14:10.710966Z 2021-12-10T17:14:06.741233Z 2021-12-10T17:14:10.949821Z 2021-12-10T17:14:06.625261Z 2021-12-10T17:14:12.370725Z 2021-12-10T17:14:07.772435Z 2021-12-10T17:14:10.829693Z 2021-12-10T17:14:06.322030Z 2021-12-10T17:14:11.298943Z 2021-12-10T17:14:04.194754Z 2021-12-10T17:14:13.366975Z 2021-12-10T17:13:59.606369Z 2021-12-10T17:14:11.709074Z 2021-12-10T17:14:00.798336Z 2021-12-10T17:14:04.535700Z 2021-12-10T17:14:01.397345Z 2021-12-10T17:14:07.879178Z 2021-12-10T17:13:59.823217Z 2021-12-10T17:14:08.723926Z 2021-12-10T17:14:04.855516Z 2021-12-10T17:14:10.816562Z 2021-12-10T17:14:03.083416Z 2021-12-10T17:14:09.230955Z 2021-12-10T17:14:02.759343Z 2021-12-10T17:14:03.453206Z 2021-12-10T17:14:04.832430Z 2021-12-10T17:13:58.358654Z 2021-12-10T17:13:56.929534Z 2021-12-10T17:13:56.774960Z 2021-12-10T17:13:56.393193Z 2021-12-10T17:13:51.066344Z 2021-12-10T17:13:59.814017Z 2021-12-10T17:14:07.516693Z 2021-12-10T17:13:59.003796Z 2021-12-10T17:14:03.278361Z 2021-12-10T17:13:55.521226Z 2021-12-10T17:14:06.846088Z 2021-12-10T17:13:51.727290Z 2021-12-10T17:14:00.924149Z 2021-12-10T17:13:51.766144Z 2021-12-10T17:14:01.056617Z 2021-12-10T17:13:51.102356Z 2021-12-10T17:13:53.541301Z 2021-12-10T17:13:53.431538Z 2021-12-10T17:13:54.285160Z 2021-12-10T17:13:48.297382Z 2021-12-10T17:14:01.243334Z 2021-12-10T17:13:43.308090Z 2021-12-10T17:13:59.351434Z 2021-12-10T17:13:37.302672Z 2021-12-10T17:13:48.406541Z 2021-12-10T17:13:39.596955Z 2021-12-10T17:13:51.771047Z 2021-12-10T17:13:40.291505Z 2021-12-10T17:13:42.640284Z 2021-12-10T17:13:27.461228Z 2021-12-10T17:13:41.974707Z 2021-12-10T17:13:22.912109Z 2021-12-10T17:13:41.972589Z 2021-12-10T17:13:22.421163Z 2021-12-10T17:13:43.032561Z 2021-12-10T17:13:10.322762Z 2021-12-10T17:13:30.730211Z 2021-12-10T17:12:56.495367Z 2021-12-10T17:12:59.565683Z 2021-12-10T17:12:55.823011Z 2021-12-10T17:12:58.912221Z 2021-12-10T17:12:54.832985Z 2021-12-10T17:12:58.629215Z 2021-12-10T17:12:56.493087Z 2021-12-10T17:12:57.430116Z 2021-12-10T17:12:54.734649Z 2021-12-10T17:12:53.155576Z 2021-12-10T17:12:52.008370Z 2021-12-10T17:12:54.767694Z 2021-12-10T17:12:49.798424Z 2021-12-10T17:12:50.764672Z 2021-12-10T17:12:48.192920Z 2021-12-10T17:12:46.653074Z 2021-12-10T17:12:46.598985Z 2021-12-10T17:12:44.075329Z 2021-12-10T17:18:31.001933Z 2021-12-10T17:18:29.467990Z 2021-12-10T17:18:26.170972Z 2021-12-10T17:18:29.069134Z 2021-12-10T17:18:26.488959Z 2021-12-10T17:18:29.828988Z 2021-12-10T17:18:24.408856Z 2021-12-10T17:18:28.674046Z 2021-12-10T17:18:27.436056Z 2021-12-10T17:18:29.996824Z 2021-12-10T17:18:25.257301Z 2021-12-10T17:18:31.001937Z 2021-12-10T17:18:26.330864Z 2021-12-10T17:18:29.397113Z 2021-12-10T17:18:27.585572Z 2021-12-10T17:18:30.706234Z 2021-12-10T17:18:22.402377Z 2021-12-10T17:18:26.501887Z 2021-12-10T17:18:27.121598Z 2021-12-10T17:18:27.216220Z 2021-12-10T17:18:26.863255Z 2021-12-10T17:18:30.089951Z 2021-12-10T17:18:26.484348Z 2021-12-10T17:18:29.245744Z 2021-12-10T17:18:24.030604Z 2021-12-10T17:18:27.697365Z 2021-12-10T17:18:25.258200Z 2021-12-10T17:18:29.864493Z 2021-12-10T17:18:18.515963Z 2021-12-10T17:18:25.335037Z 2021-12-10T17:18:19.427623Z 2021-12-10T17:18:23.728310Z 2021-12-10T17:18:21.396522Z 2021-12-10T17:18:26.827820Z 2021-12-10T17:18:25.939435Z 2021-12-10T17:18:18.834444Z 2021-12-10T17:18:22.944199Z 2021-12-10T17:18:21.929300Z 2021-12-10T17:18:22.737675Z 2021-12-10T17:18:17.153366Z 2021-12-10T17:18:21.079364Z 2021-12-10T17:18:23.507850Z 2021-12-10T17:18:19.251247Z 2021-12-10T17:18:23.666402Z 2021-12-10T17:18:25.163443Z 2021-12-10T17:18:12.751778Z 2021-12-10T17:18:18.135523Z 2021-12-10T17:18:20.812251Z 2021-12-10T17:18:16.285193Z 2021-12-10T17:18:10.089183Z 2021-12-10T17:18:16.814297Z 2021-12-10T17:18:13.436967Z 2021-12-10T17:18:12.430436Z 2021-12-10T17:18:11.238213Z 2021-12-10T17:18:20.044341Z 2021-12-10T17:18:17.479079Z 2021-12-10T17:18:05.239003Z 2021-12-10T17:18:15.403571Z 2021-12-10T17:18:14.997201Z 2021-12-10T17:18:17.908267Z 2021-12-10T17:18:11.442097Z 2021-12-10T17:18:16.593355Z 2021-12-10T17:18:06.405209Z 2021-12-10T17:18:14.587217Z 2021-12-10T17:18:12.264345Z 2021-12-10T17:18:10.991902Z 2021-12-10T17:18:07.389969Z 2021-12-10T17:18:08.121445Z 2021-12-10T17:18:03.280975Z 2021-12-10T17:18:02.282667Z 2021-12-10T17:18:02.336201Z 2021-12-10T17:18:05.138000Z 2021-12-10T17:17:58.266075Z 2021-12-10T17:18:02.698270Z 2021-12-10T17:18:01.260075Z 2021-12-10T17:17:53.193998Z 2021-12-10T17:17:51.218097Z 2021-12-10T17:17:57.676095Z 2021-12-10T17:17:57.981639Z 2021-12-10T17:17:52.526379Z 2021-12-10T17:17:53.145799Z 2021-12-10T17:17:54.810989Z 2021-12-10T17:17:50.252161Z 2021-12-10T17:17:44.261818Z 2021-12-10T17:17:19.550721Z 2021-12-10T17:17:14.433883Z 2021-12-10T17:17:19.334980Z 2021-12-10T17:17:11.991201Z 2021-12-10T17:17:16.684556Z 2021-12-10T17:17:12.631667Z 2021-12-10T17:17:15.310471Z 2021-12-10T17:17:11.075233Z 2021-12-10T17:17:12.495023Z 2021-12-10T17:17:10.164186Z 2021-12-10T17:17:11.788867Z 2021-12-10T17:17:08.862834Z 2021-12-10T17:17:12.033203Z 2021-12-10T17:17:02.101585Z 2021-12-10T17:17:00.044212Z 2021-12-10T17:16:56.735252Z 2021-12-10T17:22:49.752489Z 2021-12-10T17:22:53.259624Z 2021-12-10T17:22:52.200551Z 2021-12-10T17:22:53.336182Z 2021-12-10T17:22:51.953343Z 2021-12-10T17:22:53.142054Z 2021-12-10T17:22:51.713908Z 2021-12-10T17:22:53.260936Z 2021-12-10T17:22:50.704019Z 2021-12-10T17:22:53.192638Z 2021-12-10T17:22:49.521550Z 2021-12-10T17:22:48.681169Z 2021-12-10T17:22:44.199364Z 2021-12-10T17:22:52.224663Z 2021-12-10T17:22:51.016741Z 2021-12-10T17:22:49.849434Z 2021-12-10T17:22:50.922042Z 2021-12-10T17:22:52.900618Z 2021-12-10T17:22:50.212247Z 2021-12-10T17:22:51.336679Z 2021-12-10T17:22:50.143794Z 2021-12-10T17:22:47.318854Z 2021-12-10T17:22:47.594202Z 2021-12-10T17:22:49.972204Z 2021-12-10T17:22:49.991778Z 2021-12-10T17:22:49.661062Z 2021-12-10T17:22:44.683646Z 2021-12-10T17:22:46.965921Z 2021-12-10T17:22:45.948175Z 2021-12-10T17:22:50.961451Z 2021-12-10T17:22:48.245132Z 2021-12-10T17:22:45.022916Z 2021-12-10T17:22:43.484265Z 2021-12-10T17:22:42.897570Z 2021-12-10T17:22:51.414298Z 2021-12-10T17:22:46.204992Z 2021-12-10T17:22:46.672284Z 2021-12-10T17:22:51.228024Z 2021-12-10T17:22:47.395346Z 2021-12-10T17:22:46.779012Z 2021-12-10T17:22:48.041971Z 2021-12-10T17:22:48.611387Z 2021-12-10T17:22:48.259923Z 2021-12-10T17:22:47.948707Z 2021-12-10T17:22:49.637258Z 2021-12-10T17:22:46.558228Z 2021-12-10T17:22:43.658042Z 2021-12-10T17:22:46.667762Z 2021-12-10T17:22:40.891509Z 2021-12-10T17:22:36.915209Z 2021-12-10T17:22:45.103882Z 2021-12-10T17:22:35.413212Z 2021-12-10T17:22:32.888117Z 2021-12-10T17:22:26.953446Z 2021-12-10T17:22:34.651012Z 2021-12-10T17:22:36.727897Z 2021-12-10T17:22:33.440925Z 2021-12-10T17:22:25.442687Z 2021-12-10T17:22:29.780629Z 2021-12-10T17:22:27.378464Z 2021-12-10T17:22:28.653463Z 2021-12-10T17:22:31.280225Z 2021-12-10T17:22:20.453614Z 2021-12-10T17:22:26.956026Z 2021-12-10T17:22:29.029304Z 2021-12-10T17:22:26.524240Z 2021-12-10T17:22:28.829046Z 2021-12-10T17:22:18.536244Z 2021-12-10T17:22:21.891170Z 2021-12-10T17:22:33.604234Z 2021-12-10T17:22:22.049995Z 2021-12-10T17:22:17.062569Z 2021-12-10T17:22:17.226158Z 2021-12-10T17:22:20.656735Z 2021-12-10T17:22:14.238227Z 2021-12-10T17:22:26.241023Z 2021-12-10T17:22:08.663371Z 2021-12-10T17:22:12.934984Z 2021-12-10T17:22:04.731696Z 2021-12-10T17:22:18.086337Z 2021-12-10T17:22:16.793154Z 2021-12-10T17:22:12.688701Z 2021-12-10T17:22:02.203296Z 2021-12-10T17:22:05.709335Z 2021-12-10T17:21:47.453035Z 2021-12-10T17:21:51.421337Z 2021-12-10T17:21:54.965235Z 2021-12-10T17:22:01.629137Z 2021-12-10T17:21:54.916260Z 2021-12-10T17:21:49.141905Z 2021-12-10T17:21:30.259794Z 2021-12-10T17:21:32.633078Z 2021-12-10T17:21:30.226228Z 2021-12-10T17:21:31.516971Z 2021-12-10T17:21:29.990003Z 2021-12-10T17:21:25.385231Z 2021-12-10T17:21:22.637949Z 2021-12-10T17:21:23.981997Z 2021-12-10T17:21:20.922174Z 2021-12-10T17:21:22.494407Z", + "finished_diff": 101.277825, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 152.0592, + "mean": 51.68587149199998, + "min": 8.941699 + }, + "jobs_events_duration": { + "max": 6.111571, + "mean": 3.0055173919999976, + "min": 0.196926 + }, + "jobs_events_lag": { + "max": -1.492193, + "mean": -4.947017954000001, + "min": -8.221336 + }, + "jobs_waiting": { + "max": 41.598092, + "mean": 23.427762614000027, + "min": 0.604992 + }, + "modified": "2021-12-10T17:04:19.106522Z 2021-12-10T17:04:18.939425Z 2021-12-10T17:04:18.753379Z 2021-12-10T17:04:18.616726Z 2021-12-10T17:04:18.514165Z 2021-12-10T17:04:18.474370Z 2021-12-10T17:04:18.422077Z 2021-12-10T17:04:18.330290Z 2021-12-10T17:04:18.183401Z 2021-12-10T17:04:18.054183Z 2021-12-10T17:04:17.897023Z 2021-12-10T17:04:17.761509Z 2021-12-10T17:04:17.667018Z 2021-12-10T17:04:06.317181Z 2021-12-10T17:04:05.999365Z 2021-12-10T17:04:05.700756Z 2021-12-10T17:04:05.469562Z 2021-12-10T17:04:05.137219Z 2021-12-10T17:04:04.821354Z 2021-12-10T17:04:04.692389Z 2021-12-10T17:04:04.632784Z 2021-12-10T17:04:04.568671Z 2021-12-10T17:04:04.429734Z 2021-12-10T17:04:04.230376Z 2021-12-10T17:04:04.051313Z 2021-12-10T17:04:03.837554Z 2021-12-10T17:04:03.636090Z 2021-12-10T17:04:03.585623Z 2021-12-10T17:04:01.180322Z 2021-12-10T17:04:01.103536Z 2021-12-10T17:04:01.036959Z 2021-12-10T17:04:00.958577Z 2021-12-10T17:04:00.912508Z 2021-12-10T17:03:59.986492Z 2021-12-10T17:03:59.948021Z 2021-12-10T17:03:59.909864Z 2021-12-10T17:04:51.873426Z 2021-12-10T17:04:51.713928Z 2021-12-10T17:04:51.644814Z 2021-12-10T17:04:51.589543Z 2021-12-10T17:04:51.533380Z 2021-12-10T17:04:51.449242Z 2021-12-10T17:04:51.379311Z 2021-12-10T17:04:51.277273Z 2021-12-10T17:04:23.268185Z 2021-12-10T17:04:51.149778Z 2021-12-10T17:04:23.201310Z 2021-12-10T17:04:23.142425Z 2021-12-10T17:04:50.982936Z 2021-12-10T17:04:23.078258Z 2021-12-10T17:04:22.990160Z 2021-12-10T17:04:22.937430Z 2021-12-10T17:04:22.890291Z 2021-12-10T17:04:22.836288Z 2021-12-10T17:04:22.789267Z 2021-12-10T17:04:22.741952Z 2021-12-10T17:04:22.685209Z 2021-12-10T17:04:22.628441Z 2021-12-10T17:04:22.569286Z 2021-12-10T17:04:22.506370Z 2021-12-10T17:04:22.410656Z 2021-12-10T17:04:22.321361Z 2021-12-10T17:04:22.229333Z 2021-12-10T17:04:22.155162Z 2021-12-10T17:04:22.041290Z 2021-12-10T17:04:21.910338Z 2021-12-10T17:04:21.835849Z 2021-12-10T17:04:21.769025Z 2021-12-10T17:04:21.727603Z 2021-12-10T17:04:21.671227Z 2021-12-10T17:04:21.632267Z 2021-12-10T17:04:21.597151Z 2021-12-10T17:04:21.541300Z 2021-12-10T17:04:21.450552Z 2021-12-10T17:04:21.395975Z 2021-12-10T17:04:21.363368Z 2021-12-10T17:04:21.321858Z 2021-12-10T17:04:21.267614Z 2021-12-10T17:04:21.195770Z 2021-12-10T17:04:21.153167Z 2021-12-10T17:04:21.109043Z 2021-12-10T17:04:21.044422Z 2021-12-10T17:04:20.985198Z 2021-12-10T17:04:20.945232Z 2021-12-10T17:04:20.908007Z 2021-12-10T17:04:20.862258Z 2021-12-10T17:04:20.784171Z 2021-12-10T17:04:20.671538Z 2021-12-10T17:04:20.587589Z 2021-12-10T17:04:20.486242Z 2021-12-10T17:04:20.363520Z 2021-12-10T17:04:20.262428Z 2021-12-10T17:04:20.147424Z 2021-12-10T17:04:20.039487Z 2021-12-10T17:04:19.904508Z 2021-12-10T17:04:19.731357Z 2021-12-10T17:04:19.578302Z 2021-12-10T17:04:19.442383Z 2021-12-10T17:04:19.320857Z 2021-12-10T17:04:19.208335Z 2021-12-10T17:09:05.083680Z 2021-12-10T17:09:05.027185Z 2021-12-10T17:09:04.981384Z 2021-12-10T17:09:04.929547Z 2021-12-10T17:09:04.877097Z 2021-12-10T17:09:04.819339Z 2021-12-10T17:09:04.753170Z 2021-12-10T17:09:04.685381Z 2021-12-10T17:09:04.626859Z 2021-12-10T17:09:04.558351Z 2021-12-10T17:09:04.493220Z 2021-12-10T17:09:04.421425Z 2021-12-10T17:09:04.333336Z 2021-12-10T17:09:04.255272Z 2021-12-10T17:09:04.168316Z 2021-12-10T17:09:04.081330Z 2021-12-10T17:09:04.012358Z 2021-12-10T17:08:40.607238Z 2021-12-10T17:09:03.941204Z 2021-12-10T17:09:03.851399Z 2021-12-10T17:08:40.545184Z 2021-12-10T17:08:40.490099Z 2021-12-10T17:08:40.432170Z 2021-12-10T17:08:40.374598Z 2021-12-10T17:08:40.333874Z 2021-12-10T17:08:40.297085Z 2021-12-10T17:08:40.248279Z 2021-12-10T17:08:40.209806Z 2021-12-10T17:08:40.169764Z 2021-12-10T17:08:40.131988Z 2021-12-10T17:08:40.096485Z 2021-12-10T17:08:40.060624Z 2021-12-10T17:08:40.026336Z 2021-12-10T17:08:39.993509Z 2021-12-10T17:08:39.957963Z 2021-12-10T17:08:39.904607Z 2021-12-10T17:08:39.831222Z 2021-12-10T17:08:39.696464Z 2021-12-10T17:08:39.582190Z 2021-12-10T17:08:39.447521Z 2021-12-10T17:08:39.315251Z 2021-12-10T17:08:39.193364Z 2021-12-10T17:08:39.078255Z 2021-12-10T17:08:38.973040Z 2021-12-10T17:08:38.834448Z 2021-12-10T17:08:38.742817Z 2021-12-10T17:08:38.705354Z 2021-12-10T17:08:38.651524Z 2021-12-10T17:08:38.575370Z 2021-12-10T17:08:38.522375Z 2021-12-10T17:08:38.480903Z 2021-12-10T17:08:38.414089Z 2021-12-10T17:08:38.370639Z 2021-12-10T17:08:38.323472Z 2021-12-10T17:08:38.269424Z 2021-12-10T17:08:38.228412Z 2021-12-10T17:08:38.176460Z 2021-12-10T17:08:38.111162Z 2021-12-10T17:08:38.055066Z 2021-12-10T17:08:37.999805Z 2021-12-10T17:08:37.874703Z 2021-12-10T17:08:37.760321Z 2021-12-10T17:08:37.600582Z 2021-12-10T17:08:37.498292Z 2021-12-10T17:08:37.387031Z 2021-12-10T17:08:37.269244Z 2021-12-10T17:08:37.147375Z 2021-12-10T17:08:37.008590Z 2021-12-10T17:08:36.905412Z 2021-12-10T17:08:36.705425Z 2021-12-10T17:08:36.636026Z 2021-12-10T17:08:36.547336Z 2021-12-10T17:08:36.436415Z 2021-12-10T17:08:36.352204Z 2021-12-10T17:08:36.303929Z 2021-12-10T17:08:36.220313Z 2021-12-10T17:08:36.051635Z 2021-12-10T17:08:35.915482Z 2021-12-10T17:08:24.308147Z 2021-12-10T17:08:24.267291Z 2021-12-10T17:08:24.131239Z 2021-12-10T17:08:24.032619Z 2021-12-10T17:08:23.935559Z 2021-12-10T17:08:23.849652Z 2021-12-10T17:08:23.746210Z 2021-12-10T17:08:23.568376Z 2021-12-10T17:08:23.379325Z 2021-12-10T17:08:23.154879Z 2021-12-10T17:08:22.981395Z 2021-12-10T17:08:22.830741Z 2021-12-10T17:08:22.697961Z 2021-12-10T17:08:22.523403Z 2021-12-10T17:08:22.332885Z 2021-12-10T17:08:20.680734Z 2021-12-10T17:08:20.531955Z 2021-12-10T17:08:20.404360Z 2021-12-10T17:08:19.525453Z 2021-12-10T17:08:19.495477Z 2021-12-10T17:08:19.464224Z 2021-12-10T17:08:19.427981Z 2021-12-10T17:13:16.545580Z 2021-12-10T17:13:16.464509Z 2021-12-10T17:13:16.315985Z 2021-12-10T17:13:16.154010Z 2021-12-10T17:12:50.074327Z 2021-12-10T17:12:49.981472Z 2021-12-10T17:13:16.045253Z 2021-12-10T17:13:15.870504Z 2021-12-10T17:12:49.883703Z 2021-12-10T17:12:49.783810Z 2021-12-10T17:12:49.707287Z 2021-12-10T17:12:49.651400Z 2021-12-10T17:12:49.576587Z 2021-12-10T17:12:49.500629Z 2021-12-10T17:12:49.431380Z 2021-12-10T17:12:49.345427Z 2021-12-10T17:12:49.277392Z 2021-12-10T17:12:49.159429Z 2021-12-10T17:12:49.031531Z 2021-12-10T17:12:48.958383Z 2021-12-10T17:12:48.840007Z 2021-12-10T17:12:48.730354Z 2021-12-10T17:12:48.619468Z 2021-12-10T17:12:48.571450Z 2021-12-10T17:12:48.513781Z 2021-12-10T17:12:48.425372Z 2021-12-10T17:12:48.335371Z 2021-12-10T17:12:48.244355Z 2021-12-10T17:12:48.181774Z 2021-12-10T17:12:48.099289Z 2021-12-10T17:12:47.999291Z 2021-12-10T17:12:47.901359Z 2021-12-10T17:12:47.824464Z 2021-12-10T17:12:47.771378Z 2021-12-10T17:12:47.733313Z 2021-12-10T17:12:47.697158Z 2021-12-10T17:12:47.659515Z 2021-12-10T17:12:47.626789Z 2021-12-10T17:12:47.594845Z 2021-12-10T17:12:47.560899Z 2021-12-10T17:12:47.526137Z 2021-12-10T17:12:47.490961Z 2021-12-10T17:12:47.454515Z 2021-12-10T17:12:47.418552Z 2021-12-10T17:12:47.368488Z 2021-12-10T17:12:47.334339Z 2021-12-10T17:12:47.273401Z 2021-12-10T17:12:47.130765Z 2021-12-10T17:12:47.018089Z 2021-12-10T17:12:46.915652Z 2021-12-10T17:12:46.807972Z 2021-12-10T17:12:46.745965Z 2021-12-10T17:12:46.668158Z 2021-12-10T17:12:46.566664Z 2021-12-10T17:12:46.530812Z 2021-12-10T17:12:46.478369Z 2021-12-10T17:12:46.440960Z 2021-12-10T17:12:46.402219Z 2021-12-10T17:12:46.359647Z 2021-12-10T17:12:46.286760Z 2021-12-10T17:12:46.233404Z 2021-12-10T17:12:46.187219Z 2021-12-10T17:12:46.142945Z 2021-12-10T17:12:46.105875Z 2021-12-10T17:12:46.053638Z 2021-12-10T17:12:45.999235Z 2021-12-10T17:12:45.944144Z 2021-12-10T17:12:45.892260Z 2021-12-10T17:12:45.829238Z 2021-12-10T17:12:45.780121Z 2021-12-10T17:12:45.709435Z 2021-12-10T17:12:45.652213Z 2021-12-10T17:12:45.591210Z 2021-12-10T17:12:45.538254Z 2021-12-10T17:12:45.457334Z 2021-12-10T17:12:45.404783Z 2021-12-10T17:12:45.368841Z 2021-12-10T17:12:45.331406Z 2021-12-10T17:12:45.290402Z 2021-12-10T17:12:45.244918Z 2021-12-10T17:12:45.177562Z 2021-12-10T17:12:45.077776Z 2021-12-10T17:12:33.611046Z 2021-12-10T17:12:33.315425Z 2021-12-10T17:12:33.121927Z 2021-12-10T17:12:32.862554Z 2021-12-10T17:12:32.751623Z 2021-12-10T17:12:32.702312Z 2021-12-10T17:12:32.642203Z 2021-12-10T17:12:32.551638Z 2021-12-10T17:12:32.414246Z 2021-12-10T17:12:32.283888Z 2021-12-10T17:12:32.203600Z 2021-12-10T17:12:32.047622Z 2021-12-10T17:12:29.939829Z 2021-12-10T17:12:29.885245Z 2021-12-10T17:12:29.841239Z 2021-12-10T17:12:29.797280Z 2021-12-10T17:12:29.747044Z 2021-12-10T17:12:29.268592Z 2021-12-10T17:17:36.180549Z 2021-12-10T17:17:03.415109Z 2021-12-10T17:17:03.369446Z 2021-12-10T17:17:03.325109Z 2021-12-10T17:17:03.274123Z 2021-12-10T17:17:03.227900Z 2021-12-10T17:17:03.184217Z 2021-12-10T17:17:03.150097Z 2021-12-10T17:17:03.116770Z 2021-12-10T17:17:03.078647Z 2021-12-10T17:17:03.042171Z 2021-12-10T17:17:02.999083Z 2021-12-10T17:17:02.962140Z 2021-12-10T17:17:02.928316Z 2021-12-10T17:17:02.887665Z 2021-12-10T17:17:02.844662Z 2021-12-10T17:17:02.807535Z 2021-12-10T17:17:02.773653Z 2021-12-10T17:17:02.740184Z 2021-12-10T17:17:02.701729Z 2021-12-10T17:17:02.664461Z 2021-12-10T17:17:02.627909Z 2021-12-10T17:17:02.585343Z 2021-12-10T17:17:02.540157Z 2021-12-10T17:17:02.479502Z 2021-12-10T17:17:02.425677Z 2021-12-10T17:17:02.384052Z 2021-12-10T17:17:02.345951Z 2021-12-10T17:17:02.308383Z 2021-12-10T17:17:02.272620Z 2021-12-10T17:17:02.237179Z 2021-12-10T17:17:02.202699Z 2021-12-10T17:17:02.165106Z 2021-12-10T17:17:02.128191Z 2021-12-10T17:17:02.086397Z 2021-12-10T17:17:02.047405Z 2021-12-10T17:17:02.010178Z 2021-12-10T17:17:01.972679Z 2021-12-10T17:17:01.937493Z 2021-12-10T17:17:01.902840Z 2021-12-10T17:17:01.868003Z 2021-12-10T17:17:01.828299Z 2021-12-10T17:17:01.783181Z 2021-12-10T17:17:01.734159Z 2021-12-10T17:17:01.679402Z 2021-12-10T17:17:01.626934Z 2021-12-10T17:17:01.534389Z 2021-12-10T17:17:01.458164Z 2021-12-10T17:17:01.369171Z 2021-12-10T17:17:01.285206Z 2021-12-10T17:17:01.238589Z 2021-12-10T17:17:01.200135Z 2021-12-10T17:17:01.142239Z 2021-12-10T17:17:01.034473Z 2021-12-10T17:17:01.073890Z 2021-12-10T17:17:00.996087Z 2021-12-10T17:17:00.952686Z 2021-12-10T17:17:00.915293Z 2021-12-10T17:17:00.876893Z 2021-12-10T17:17:00.787364Z 2021-12-10T17:17:00.689164Z 2021-12-10T17:17:00.602187Z 2021-12-10T17:17:00.545519Z 2021-12-10T17:17:00.411279Z 2021-12-10T17:17:00.346356Z 2021-12-10T17:17:00.301829Z 2021-12-10T17:17:00.266984Z 2021-12-10T17:17:00.230434Z 2021-12-10T17:17:00.189141Z 2021-12-10T17:17:00.151505Z 2021-12-10T17:17:00.102956Z 2021-12-10T17:17:00.032032Z 2021-12-10T17:16:59.984453Z 2021-12-10T17:16:59.926090Z 2021-12-10T17:16:59.842094Z 2021-12-10T17:16:59.793618Z 2021-12-10T17:16:59.727451Z 2021-12-10T17:16:59.666306Z 2021-12-10T17:16:59.615560Z 2021-12-10T17:16:59.538853Z 2021-12-10T17:16:59.499653Z 2021-12-10T17:16:59.459856Z 2021-12-10T17:16:59.417640Z 2021-12-10T17:16:59.352436Z 2021-12-10T17:16:51.595960Z 2021-12-10T17:16:51.293014Z 2021-12-10T17:16:50.996125Z 2021-12-10T17:16:50.866582Z 2021-12-10T17:16:50.706564Z 2021-12-10T17:16:50.358532Z 2021-12-10T17:16:50.114730Z 2021-12-10T17:16:49.892441Z 2021-12-10T17:16:49.720802Z 2021-12-10T17:16:49.457542Z 2021-12-10T17:16:49.326104Z 2021-12-10T17:16:49.220909Z 2021-12-10T17:16:48.970999Z 2021-12-10T17:16:46.749500Z 2021-12-10T17:16:46.613256Z 2021-12-10T17:16:45.595573Z 2021-12-10T17:21:42.682407Z 2021-12-10T17:21:42.569244Z 2021-12-10T17:21:42.444211Z 2021-12-10T17:21:42.334604Z 2021-12-10T17:21:42.267832Z 2021-12-10T17:21:42.204134Z 2021-12-10T17:21:42.152185Z 2021-12-10T17:21:42.079532Z 2021-12-10T17:21:41.986479Z 2021-12-10T17:21:41.910333Z 2021-12-10T17:21:41.839154Z 2021-12-10T17:21:41.748625Z 2021-12-10T17:21:41.656328Z 2021-12-10T17:21:41.578400Z 2021-12-10T17:21:41.500132Z 2021-12-10T17:21:41.409198Z 2021-12-10T17:21:41.335321Z 2021-12-10T17:21:41.222872Z 2021-12-10T17:21:41.135203Z 2021-12-10T17:21:41.022085Z 2021-12-10T17:21:40.872317Z 2021-12-10T17:21:40.752773Z 2021-12-10T17:21:40.653221Z 2021-12-10T17:21:40.555444Z 2021-12-10T17:21:40.481164Z 2021-12-10T17:21:40.410325Z 2021-12-10T17:21:40.337408Z 2021-12-10T17:21:40.256306Z 2021-12-10T17:21:40.165188Z 2021-12-10T17:21:40.051203Z 2021-12-10T17:21:39.963327Z 2021-12-10T17:21:39.864206Z 2021-12-10T17:21:39.753452Z 2021-12-10T17:21:39.634318Z 2021-12-10T17:21:39.528146Z 2021-12-10T17:21:39.449282Z 2021-12-10T17:21:39.366221Z 2021-12-10T17:21:39.281158Z 2021-12-10T17:21:39.203672Z 2021-12-10T17:21:39.131386Z 2021-12-10T17:21:39.056244Z 2021-12-10T17:21:38.981188Z 2021-12-10T17:21:38.898221Z 2021-12-10T17:21:38.797252Z 2021-12-10T17:21:38.678210Z 2021-12-10T17:21:38.599126Z 2021-12-10T17:21:38.464872Z 2021-12-10T17:21:38.338405Z 2021-12-10T17:21:23.907990Z 2021-12-10T17:21:23.870979Z 2021-12-10T17:21:38.241453Z 2021-12-10T17:21:23.837537Z 2021-12-10T17:21:23.805791Z 2021-12-10T17:21:23.774279Z 2021-12-10T17:21:23.741677Z 2021-12-10T17:21:23.705314Z 2021-12-10T17:21:23.652072Z 2021-12-10T17:21:23.612633Z 2021-12-10T17:21:23.579651Z 2021-12-10T17:21:23.545854Z 2021-12-10T17:21:23.513290Z 2021-12-10T17:21:23.477739Z 2021-12-10T17:21:23.443748Z 2021-12-10T17:21:23.408086Z 2021-12-10T17:21:23.371967Z 2021-12-10T17:21:23.334612Z 2021-12-10T17:21:23.302022Z 2021-12-10T17:21:23.205494Z 2021-12-10T17:21:23.265824Z 2021-12-10T17:21:23.099005Z 2021-12-10T17:21:23.031533Z 2021-12-10T17:21:22.975668Z 2021-12-10T17:21:22.811389Z 2021-12-10T17:21:22.622691Z 2021-12-10T17:21:22.504519Z 2021-12-10T17:21:22.366574Z 2021-12-10T17:21:22.272169Z 2021-12-10T17:21:22.161562Z 2021-12-10T17:21:22.001979Z 2021-12-10T17:21:21.846994Z 2021-12-10T17:21:21.697795Z 2021-12-10T17:21:21.584513Z 2021-12-10T17:21:21.368530Z 2021-12-10T17:21:20.744485Z 2021-12-10T17:21:20.621553Z 2021-12-10T17:21:20.434901Z 2021-12-10T17:21:20.343082Z 2021-12-10T17:21:20.255835Z 2021-12-10T17:21:19.980037Z 2021-12-10T17:21:19.884682Z 2021-12-10T17:21:12.855848Z 2021-12-10T17:21:12.614577Z 2021-12-10T17:21:12.432962Z 2021-12-10T17:21:12.150039Z 2021-12-10T17:21:11.985216Z 2021-12-10T17:21:09.426342Z 2021-12-10T17:21:09.378319Z 2021-12-10T17:21:08.819786Z 2021-12-10T17:21:08.780360Z 2021-12-10T17:21:08.311869Z", + "modified_diff": 45.9703526, + "started": "2021-12-10T17:04:26.778695Z 2021-12-10T17:04:26.473167Z 2021-12-10T17:04:26.373982Z 2021-12-10T17:04:25.987972Z 2021-12-10T17:04:25.818988Z 2021-12-10T17:04:25.450373Z 2021-12-10T17:04:25.147177Z 2021-12-10T17:04:24.926138Z 2021-12-10T17:04:24.603134Z 2021-12-10T17:04:24.449181Z 2021-12-10T17:04:24.023934Z 2021-12-10T17:04:23.820134Z 2021-12-10T17:04:23.665874Z 2021-12-10T17:04:13.459415Z 2021-12-10T17:04:12.614015Z 2021-12-10T17:04:12.939994Z 2021-12-10T17:04:11.196519Z 2021-12-10T17:04:11.499778Z 2021-12-10T17:04:10.340915Z 2021-12-10T17:04:10.757809Z 2021-12-10T17:04:09.655087Z 2021-12-10T17:04:09.174184Z 2021-12-10T17:04:08.562712Z 2021-12-10T17:04:08.248304Z 2021-12-10T17:04:07.833301Z 2021-12-10T17:04:08.013289Z 2021-12-10T17:04:07.314914Z 2021-12-10T17:04:07.134472Z 2021-12-10T17:04:02.431401Z 2021-12-10T17:04:02.217148Z 2021-12-10T17:04:01.909524Z 2021-12-10T17:04:01.713161Z 2021-12-10T17:04:01.467394Z 2021-12-10T17:04:00.371602Z 2021-12-10T17:04:00.278493Z 2021-12-10T17:04:00.195632Z 2021-12-10T17:04:54.014093Z 2021-12-10T17:04:54.402218Z 2021-12-10T17:04:54.282178Z 2021-12-10T17:04:53.741769Z 2021-12-10T17:04:53.515191Z 2021-12-10T17:04:53.243210Z 2021-12-10T17:04:52.980546Z 2021-12-10T17:04:52.728280Z 2021-12-10T17:04:48.930114Z 2021-12-10T17:04:52.496038Z 2021-12-10T17:04:48.589993Z 2021-12-10T17:04:47.904308Z 2021-12-10T17:04:52.414243Z 2021-12-10T17:04:47.825684Z 2021-12-10T17:04:47.444850Z 2021-12-10T17:04:46.732922Z 2021-12-10T17:04:46.229845Z 2021-12-10T17:04:46.005944Z 2021-12-10T17:04:45.547285Z 2021-12-10T17:04:45.353619Z 2021-12-10T17:04:44.764137Z 2021-12-10T17:04:44.237363Z 2021-12-10T17:04:43.739193Z 2021-12-10T17:04:43.328580Z 2021-12-10T17:04:42.450190Z 2021-12-10T17:04:42.285277Z 2021-12-10T17:04:41.889533Z 2021-12-10T17:04:41.719211Z 2021-12-10T17:04:40.978207Z 2021-12-10T17:04:40.845678Z 2021-12-10T17:04:40.529767Z 2021-12-10T17:04:40.496083Z 2021-12-10T17:04:39.717899Z 2021-12-10T17:04:39.553216Z 2021-12-10T17:04:38.661126Z 2021-12-10T17:04:38.482212Z 2021-12-10T17:04:37.783979Z 2021-12-10T17:04:37.229602Z 2021-12-10T17:04:36.690109Z 2021-12-10T17:04:36.393279Z 2021-12-10T17:04:35.689193Z 2021-12-10T17:04:35.293610Z 2021-12-10T17:04:34.610202Z 2021-12-10T17:04:34.543406Z 2021-12-10T17:04:33.809104Z 2021-12-10T17:04:33.655539Z 2021-12-10T17:04:32.899261Z 2021-12-10T17:04:32.511671Z 2021-12-10T17:04:32.153099Z 2021-12-10T17:04:31.922866Z 2021-12-10T17:04:31.622174Z 2021-12-10T17:04:31.526928Z 2021-12-10T17:04:31.140966Z 2021-12-10T17:04:30.886164Z 2021-12-10T17:04:30.429189Z 2021-12-10T17:04:30.929943Z 2021-12-10T17:04:29.946369Z 2021-12-10T17:04:30.547818Z 2021-12-10T17:04:29.184085Z 2021-12-10T17:04:28.825215Z 2021-12-10T17:04:28.342381Z 2021-12-10T17:04:27.853329Z 2021-12-10T17:04:27.328179Z 2021-12-10T17:04:26.944708Z 2021-12-10T17:09:10.798458Z 2021-12-10T17:09:10.531462Z 2021-12-10T17:09:09.952846Z 2021-12-10T17:09:09.905510Z 2021-12-10T17:09:09.325734Z 2021-12-10T17:09:09.223614Z 2021-12-10T17:09:08.683630Z 2021-12-10T17:09:08.539743Z 2021-12-10T17:09:07.895991Z 2021-12-10T17:09:07.558063Z 2021-12-10T17:09:07.256082Z 2021-12-10T17:09:07.184396Z 2021-12-10T17:09:06.694039Z 2021-12-10T17:09:06.573449Z 2021-12-10T17:09:06.311092Z 2021-12-10T17:09:06.074879Z 2021-12-10T17:09:05.906634Z 2021-12-10T17:09:01.734135Z 2021-12-10T17:09:05.712773Z 2021-12-10T17:09:05.627438Z 2021-12-10T17:09:01.454183Z 2021-12-10T17:09:00.877298Z 2021-12-10T17:09:00.500364Z 2021-12-10T17:09:00.006217Z 2021-12-10T17:08:59.718035Z 2021-12-10T17:08:59.681046Z 2021-12-10T17:08:58.932167Z 2021-12-10T17:08:58.735174Z 2021-12-10T17:08:58.203628Z 2021-12-10T17:08:58.001765Z 2021-12-10T17:08:57.678122Z 2021-12-10T17:08:57.234795Z 2021-12-10T17:08:57.005498Z 2021-12-10T17:08:56.901444Z 2021-12-10T17:08:56.173777Z 2021-12-10T17:08:55.440044Z 2021-12-10T17:08:55.259439Z 2021-12-10T17:08:54.841641Z 2021-12-10T17:08:54.454130Z 2021-12-10T17:08:54.017139Z 2021-12-10T17:08:53.694083Z 2021-12-10T17:08:53.196829Z 2021-12-10T17:08:52.739046Z 2021-12-10T17:08:52.843185Z 2021-12-10T17:08:52.055728Z 2021-12-10T17:08:51.680595Z 2021-12-10T17:08:51.514282Z 2021-12-10T17:08:50.855414Z 2021-12-10T17:08:50.594351Z 2021-12-10T17:08:50.429396Z 2021-12-10T17:08:49.947772Z 2021-12-10T17:08:49.398972Z 2021-12-10T17:08:48.818834Z 2021-12-10T17:08:48.568250Z 2021-12-10T17:08:48.308143Z 2021-12-10T17:08:48.015534Z 2021-12-10T17:08:47.462962Z 2021-12-10T17:08:47.324489Z 2021-12-10T17:08:46.651312Z 2021-12-10T17:08:46.323180Z 2021-12-10T17:08:46.003247Z 2021-12-10T17:08:45.679296Z 2021-12-10T17:08:45.479089Z 2021-12-10T17:08:45.109419Z 2021-12-10T17:08:44.736854Z 2021-12-10T17:08:44.246238Z 2021-12-10T17:08:44.212444Z 2021-12-10T17:08:43.473615Z 2021-12-10T17:08:43.175168Z 2021-12-10T17:08:42.842900Z 2021-12-10T17:08:42.521162Z 2021-12-10T17:08:42.147909Z 2021-12-10T17:08:41.836776Z 2021-12-10T17:08:41.788403Z 2021-12-10T17:08:41.525401Z 2021-12-10T17:08:41.448828Z 2021-12-10T17:08:41.263357Z 2021-12-10T17:08:41.074170Z 2021-12-10T17:08:33.273188Z 2021-12-10T17:08:32.419801Z 2021-12-10T17:08:32.980399Z 2021-12-10T17:08:31.744329Z 2021-12-10T17:08:31.290335Z 2021-12-10T17:08:30.223703Z 2021-12-10T17:08:29.646009Z 2021-12-10T17:08:28.304229Z 2021-12-10T17:08:28.244968Z 2021-12-10T17:08:28.257888Z 2021-12-10T17:08:27.164748Z 2021-12-10T17:08:26.686251Z 2021-12-10T17:08:26.195621Z 2021-12-10T17:08:26.173908Z 2021-12-10T17:08:25.109783Z 2021-12-10T17:08:21.756922Z 2021-12-10T17:08:21.483137Z 2021-12-10T17:08:21.947036Z 2021-12-10T17:08:19.993205Z 2021-12-10T17:08:19.878520Z 2021-12-10T17:08:19.778355Z 2021-12-10T17:08:19.689526Z 2021-12-10T17:13:18.395272Z 2021-12-10T17:13:18.139561Z 2021-12-10T17:13:17.740973Z 2021-12-10T17:13:17.635436Z 2021-12-10T17:13:13.713044Z 2021-12-10T17:13:13.559994Z 2021-12-10T17:13:17.307244Z 2021-12-10T17:13:17.163026Z 2021-12-10T17:13:13.097111Z 2021-12-10T17:13:12.750508Z 2021-12-10T17:13:12.367160Z 2021-12-10T17:13:12.288339Z 2021-12-10T17:13:11.805081Z 2021-12-10T17:13:11.994421Z 2021-12-10T17:13:11.346277Z 2021-12-10T17:13:11.505131Z 2021-12-10T17:13:10.880855Z 2021-12-10T17:13:10.635332Z 2021-12-10T17:13:10.099045Z 2021-12-10T17:13:10.016708Z 2021-12-10T17:13:09.503024Z 2021-12-10T17:13:09.352984Z 2021-12-10T17:13:08.890543Z 2021-12-10T17:13:08.535168Z 2021-12-10T17:13:08.280810Z 2021-12-10T17:13:08.067620Z 2021-12-10T17:13:07.633178Z 2021-12-10T17:13:07.546364Z 2021-12-10T17:13:07.057551Z 2021-12-10T17:13:06.782708Z 2021-12-10T17:13:06.507484Z 2021-12-10T17:13:06.247488Z 2021-12-10T17:13:05.877228Z 2021-12-10T17:13:05.683943Z 2021-12-10T17:13:05.129129Z 2021-12-10T17:13:05.024410Z 2021-12-10T17:13:04.408838Z 2021-12-10T17:13:04.102662Z 2021-12-10T17:13:03.804636Z 2021-12-10T17:13:03.717411Z 2021-12-10T17:13:03.416224Z 2021-12-10T17:13:02.638284Z 2021-12-10T17:13:02.584125Z 2021-12-10T17:13:02.160573Z 2021-12-10T17:13:01.790103Z 2021-12-10T17:13:01.434235Z 2021-12-10T17:13:01.119296Z 2021-12-10T17:13:00.748243Z 2021-12-10T17:13:00.247306Z 2021-12-10T17:12:59.770751Z 2021-12-10T17:12:59.411257Z 2021-12-10T17:12:59.068059Z 2021-12-10T17:12:58.953683Z 2021-12-10T17:12:58.894169Z 2021-12-10T17:12:58.531261Z 2021-12-10T17:12:58.529623Z 2021-12-10T17:12:57.992819Z 2021-12-10T17:12:57.615804Z 2021-12-10T17:12:57.305979Z 2021-12-10T17:12:57.288588Z 2021-12-10T17:12:56.776704Z 2021-12-10T17:12:56.536992Z 2021-12-10T17:12:56.270580Z 2021-12-10T17:12:56.155598Z 2021-12-10T17:12:55.881457Z 2021-12-10T17:12:55.685483Z 2021-12-10T17:12:55.235222Z 2021-12-10T17:12:54.736778Z 2021-12-10T17:12:54.180027Z 2021-12-10T17:12:54.151330Z 2021-12-10T17:12:53.368161Z 2021-12-10T17:12:53.313734Z 2021-12-10T17:12:52.932890Z 2021-12-10T17:12:52.528069Z 2021-12-10T17:12:52.293021Z 2021-12-10T17:12:51.979290Z 2021-12-10T17:12:51.734183Z 2021-12-10T17:12:51.458011Z 2021-12-10T17:12:51.108370Z 2021-12-10T17:12:50.967360Z 2021-12-10T17:12:50.726516Z 2021-12-10T17:12:50.590955Z 2021-12-10T17:12:42.003297Z 2021-12-10T17:12:41.250208Z 2021-12-10T17:12:40.560276Z 2021-12-10T17:12:40.229338Z 2021-12-10T17:12:38.762431Z 2021-12-10T17:12:39.087190Z 2021-12-10T17:12:37.805528Z 2021-12-10T17:12:37.176816Z 2021-12-10T17:12:36.066444Z 2021-12-10T17:12:35.263138Z 2021-12-10T17:12:34.552506Z 2021-12-10T17:12:34.402589Z 2021-12-10T17:12:31.262493Z 2021-12-10T17:12:31.137440Z 2021-12-10T17:12:30.756748Z 2021-12-10T17:12:30.556768Z 2021-12-10T17:12:30.442165Z 2021-12-10T17:12:29.579024Z 2021-12-10T17:17:36.671958Z 2021-12-10T17:17:32.361097Z 2021-12-10T17:17:32.211138Z 2021-12-10T17:17:31.736953Z 2021-12-10T17:17:31.541253Z 2021-12-10T17:17:31.079106Z 2021-12-10T17:17:30.791296Z 2021-12-10T17:17:30.540720Z 2021-12-10T17:17:30.229532Z 2021-12-10T17:17:29.969894Z 2021-12-10T17:17:29.396691Z 2021-12-10T17:17:28.960134Z 2021-12-10T17:17:28.363992Z 2021-12-10T17:17:28.362599Z 2021-12-10T17:17:27.700060Z 2021-12-10T17:17:27.278668Z 2021-12-10T17:17:27.037116Z 2021-12-10T17:17:26.626024Z 2021-12-10T17:17:26.262939Z 2021-12-10T17:17:25.757975Z 2021-12-10T17:17:25.462307Z 2021-12-10T17:17:25.181076Z 2021-12-10T17:17:24.627308Z 2021-12-10T17:17:24.542080Z 2021-12-10T17:17:24.105299Z 2021-12-10T17:17:23.653198Z 2021-12-10T17:17:23.130953Z 2021-12-10T17:17:22.945633Z 2021-12-10T17:17:22.391253Z 2021-12-10T17:17:22.106662Z 2021-12-10T17:17:22.056087Z 2021-12-10T17:17:21.721122Z 2021-12-10T17:17:21.300144Z 2021-12-10T17:17:21.011026Z 2021-12-10T17:17:20.730254Z 2021-12-10T17:17:20.102123Z 2021-12-10T17:17:19.826735Z 2021-12-10T17:17:19.560304Z 2021-12-10T17:17:19.180183Z 2021-12-10T17:17:18.785380Z 2021-12-10T17:17:18.483667Z 2021-12-10T17:17:18.167507Z 2021-12-10T17:17:17.722360Z 2021-12-10T17:17:17.476257Z 2021-12-10T17:17:17.213047Z 2021-12-10T17:17:16.832375Z 2021-12-10T17:17:16.412199Z 2021-12-10T17:17:16.170541Z 2021-12-10T17:17:15.662292Z 2021-12-10T17:17:15.153014Z 2021-12-10T17:17:15.046280Z 2021-12-10T17:17:14.493012Z 2021-12-10T17:17:14.291796Z 2021-12-10T17:17:13.102956Z 2021-12-10T17:17:13.932645Z 2021-12-10T17:17:12.921877Z 2021-12-10T17:17:12.033301Z 2021-12-10T17:17:11.981958Z 2021-12-10T17:17:11.035162Z 2021-12-10T17:17:10.663455Z 2021-12-10T17:17:10.156006Z 2021-12-10T17:17:09.747217Z 2021-12-10T17:17:09.372436Z 2021-12-10T17:17:08.929368Z 2021-12-10T17:17:08.837527Z 2021-12-10T17:17:08.867097Z 2021-12-10T17:17:08.269360Z 2021-12-10T17:17:07.885113Z 2021-12-10T17:17:07.498993Z 2021-12-10T17:17:07.180430Z 2021-12-10T17:17:07.210166Z 2021-12-10T17:17:07.057778Z 2021-12-10T17:17:06.435222Z 2021-12-10T17:17:06.249323Z 2021-12-10T17:17:05.963824Z 2021-12-10T17:17:05.800308Z 2021-12-10T17:17:05.506932Z 2021-12-10T17:17:04.991209Z 2021-12-10T17:17:04.873280Z 2021-12-10T17:17:04.597227Z 2021-12-10T17:17:04.470752Z 2021-12-10T17:17:03.989021Z 2021-12-10T17:17:04.016323Z 2021-12-10T17:17:03.870259Z 2021-12-10T17:16:57.456324Z 2021-12-10T17:16:57.365227Z 2021-12-10T17:16:56.998750Z 2021-12-10T17:16:56.974559Z 2021-12-10T17:16:56.276304Z 2021-12-10T17:16:56.180336Z 2021-12-10T17:16:55.741925Z 2021-12-10T17:16:55.335675Z 2021-12-10T17:16:54.731558Z 2021-12-10T17:16:54.295121Z 2021-12-10T17:16:54.105348Z 2021-12-10T17:16:53.182344Z 2021-12-10T17:16:52.865242Z 2021-12-10T17:16:47.778858Z 2021-12-10T17:16:47.710792Z 2021-12-10T17:16:45.790001Z 2021-12-10T17:22:00.218285Z 2021-12-10T17:22:00.121746Z 2021-12-10T17:21:59.679945Z 2021-12-10T17:21:59.278144Z 2021-12-10T17:21:58.990347Z 2021-12-10T17:21:58.265923Z 2021-12-10T17:21:58.326169Z 2021-12-10T17:21:57.750085Z 2021-12-10T17:21:57.670497Z 2021-12-10T17:21:57.149843Z 2021-12-10T17:21:56.721738Z 2021-12-10T17:21:56.310178Z 2021-12-10T17:21:56.210237Z 2021-12-10T17:21:55.525432Z 2021-12-10T17:21:55.098811Z 2021-12-10T17:21:54.727160Z 2021-12-10T17:21:54.572186Z 2021-12-10T17:21:53.635140Z 2021-12-10T17:21:53.528897Z 2021-12-10T17:21:52.779229Z 2021-12-10T17:21:52.669244Z 2021-12-10T17:21:52.390230Z 2021-12-10T17:21:52.329122Z 2021-12-10T17:21:51.282918Z 2021-12-10T17:21:51.373139Z 2021-12-10T17:21:50.456600Z 2021-12-10T17:21:50.019435Z 2021-12-10T17:21:49.560196Z 2021-12-10T17:21:49.075606Z 2021-12-10T17:21:49.024679Z 2021-12-10T17:21:48.849427Z 2021-12-10T17:21:48.284199Z 2021-12-10T17:21:47.998088Z 2021-12-10T17:21:47.678153Z 2021-12-10T17:21:47.633332Z 2021-12-10T17:21:47.230088Z 2021-12-10T17:21:46.684366Z 2021-12-10T17:21:46.133141Z 2021-12-10T17:21:45.678253Z 2021-12-10T17:21:45.560701Z 2021-12-10T17:21:45.298919Z 2021-12-10T17:21:44.870453Z 2021-12-10T17:21:44.463228Z 2021-12-10T17:21:44.336712Z 2021-12-10T17:21:43.970197Z 2021-12-10T17:21:43.694015Z 2021-12-10T17:21:43.564222Z 2021-12-10T17:21:43.324208Z 2021-12-10T17:21:35.477525Z 2021-12-10T17:21:35.248474Z 2021-12-10T17:21:43.197669Z 2021-12-10T17:21:34.818399Z 2021-12-10T17:21:34.396715Z 2021-12-10T17:21:34.230531Z 2021-12-10T17:21:33.401552Z 2021-12-10T17:21:33.093849Z 2021-12-10T17:21:32.915319Z 2021-12-10T17:21:32.839299Z 2021-12-10T17:21:32.318489Z 2021-12-10T17:21:32.338609Z 2021-12-10T17:21:31.804151Z 2021-12-10T17:21:31.599206Z 2021-12-10T17:21:31.419992Z 2021-12-10T17:21:31.205224Z 2021-12-10T17:21:31.081477Z 2021-12-10T17:21:30.864968Z 2021-12-10T17:21:30.499312Z 2021-12-10T17:21:29.711854Z 2021-12-10T17:21:30.143097Z 2021-12-10T17:21:29.642765Z 2021-12-10T17:21:28.922040Z 2021-12-10T17:21:28.650702Z 2021-12-10T17:21:28.369728Z 2021-12-10T17:21:28.050032Z 2021-12-10T17:21:27.607517Z 2021-12-10T17:21:27.347340Z 2021-12-10T17:21:26.948384Z 2021-12-10T17:21:26.755920Z 2021-12-10T17:21:26.546505Z 2021-12-10T17:21:26.509500Z 2021-12-10T17:21:26.091175Z 2021-12-10T17:21:25.879091Z 2021-12-10T17:21:25.914277Z 2021-12-10T17:21:25.451179Z 2021-12-10T17:21:25.257954Z 2021-12-10T17:21:24.899037Z 2021-12-10T17:21:24.790295Z 2021-12-10T17:21:24.541369Z 2021-12-10T17:21:24.514135Z 2021-12-10T17:21:24.263188Z 2021-12-10T17:21:15.567505Z 2021-12-10T17:21:15.560940Z 2021-12-10T17:21:15.387508Z 2021-12-10T17:21:14.111512Z 2021-12-10T17:21:14.445117Z 2021-12-10T17:21:09.767931Z 2021-12-10T17:21:09.601034Z 2021-12-10T17:21:09.080710Z 2021-12-10T17:21:09.023809Z 2021-12-10T17:21:08.586839Z", + "started_diff": 51.329033800000005, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 93, 92, 91, 89, 88, 87, 86, 84, 83, 82, 81, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 66, 65, 64, 63, 62, 61, 60, 169, 168, 167, 166, 165, 164, 163, 162, 160, 159, 158, 157, 156, 155, 154, 152, 151, 150, 148, 147, 146, 145, 144, 142, 141, 140, 139, 138, 136, 135, 134, 133, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 171, 170, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 271, 270, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 371, 370, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, 471, 470]" + }, + "started": "2021-12-10T17:47:38.240658+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-api.json new file mode 100644 index 00000000..cc7bd633 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-17T16:19:14.606423+00:00", + "golden": "true", + "id": "run-2021-12-17T11_38_32_937_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2409.166666666667, + "max": 25035.933333333334, + "mean": 5245.777161052861, + "median": 3226.0, + "min": 1344.5333333333333, + "non_zero_mean": 5245.777161052861, + "non_zero_median": 3226.0, + "percentile25": 2711.5166666666664, + "percentile75": 5120.683333333333, + "percentile90": 12346.280000000012, + "percentile99": 22788.537999999997, + "percentile999": 24769.245266666687, + "range": 23691.4, + "samples": 300, + "stdev": 4742.995847331315, + "sum": 1573733.148315858 + } + }, + "cpu": { + "system": { + "iqr": 0.22600000000000192, + "max": 2.3579999999999925, + "mean": 0.36853557367798967, + "median": 0.1490000000000028, + "min": 0.04733333333333576, + "non_zero_mean": 0.36853557367798967, + "non_zero_median": 0.1490000000000028, + "percentile25": 0.0814999999999884, + "percentile75": 0.30749999999999034, + "percentile90": 1.2379999999999913, + "percentile99": 2.151659999999999, + "percentile999": 2.3542126666666627, + "range": 2.310666666666657, + "samples": 300, + "stdev": 0.522719257592032, + "sum": 110.56067210339688 + }, + "user": { + "iqr": 0.5606666666666266, + "max": 6.016666666666667, + "mean": 1.386182280115016, + "median": 0.8006666666666813, + "min": 0.32066666666661753, + "non_zero_mean": 1.386182280115016, + "non_zero_median": 0.8006666666666813, + "percentile25": 0.6173333333333707, + "percentile75": 1.1779999999999973, + "percentile90": 3.9547333333333388, + "percentile99": 5.828146666666656, + "percentile999": 5.971417999999999, + "range": 5.6960000000000495, + "samples": 300, + "stdev": 1.4156892737402316, + "sum": 415.85468403450466 + } + }, + "entropy_available_bits": { + "iqr": 1.7167066640001778, + "max": 214.13333333333335, + "mean": 20.251111304338774, + "median": 1.7000000000000002, + "min": 0.2, + "non_zero_mean": 20.251111304338774, + "non_zero_median": 1.7000000000000002, + "percentile25": 0.7832933359998222, + "percentile75": 2.5, + "percentile90": 19.54000000000297, + "percentile99": 213.60399999999998, + "percentile999": 214.09346666666667, + "range": 213.93333333333337, + "samples": 300, + "stdev": 56.447691278373355, + "sum": 6075.333391301642 + }, + "memory": { + "used": { + "iqr": 350903296.0, + "max": 15055134720.0, + "mean": 6341388151.466666, + "median": 5865347072.0, + "min": 5369720832.0, + "non_zero_mean": 6341388151.466666, + "non_zero_median": 5865347072.0, + "percentile25": 5749081088.0, + "percentile75": 6099984384.0, + "percentile90": 7168719257.6, + "percentile99": 13262096793.599993, + "percentile999": 14848512458.752014, + "range": 9685413888.0, + "samples": 300, + "stdev": 1518359405.8273644, + "sum": 1902416445440.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.5333333333333333, + "mean": 0.0022222222222222222, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.3333333333333333, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.41373333333334206, + "range": 0.5333333333333333, + "samples": 300, + "stdev": 0.03171469494393873, + "sum": 0.6666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 77.05, + "max": 1448.6, + "mean": 117.49512052588304, + "median": 25.93333333333333, + "min": 0.0, + "non_zero_mean": 118.28367838176145, + "non_zero_median": 26.199999999999996, + "percentile25": 1.6333333333333333, + "percentile75": 78.68333333333334, + "percentile90": 348.6666666666673, + "percentile99": 1289.8586666666665, + "percentile999": 1421.3710666666686, + "range": 1448.6, + "samples": 300, + "stdev": 253.85443253562534, + "sum": 35248.53615776491 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 870.7833333333333, + "max": 2896.266666666667, + "mean": 1536.0771506350256, + "median": 1432.6666666666665, + "min": 442.6666666666667, + "non_zero_mean": 1536.0771506350256, + "non_zero_median": 1432.6666666666665, + "percentile25": 1122.2833333333333, + "percentile75": 1993.0666666666666, + "percentile90": 2284.1133333333337, + "percentile99": 2727.0726666666665, + "percentile999": 2874.5592666666685, + "range": 2453.6000000000004, + "samples": 300, + "stdev": 555.2658361914017, + "sum": 460823.1451905079 + } + }, + "cpu": { + "system": { + "iqr": 0.05366666666666807, + "max": 0.3293333333333369, + "mean": 0.10210000907571744, + "median": 0.08800000000000144, + "min": 0.017333333333328937, + "non_zero_mean": 0.10210000907571744, + "non_zero_median": 0.08800000000000144, + "percentile25": 0.06383333333333117, + "percentile75": 0.11749999999999924, + "percentile90": 0.1873333333333335, + "percentile99": 0.2620266666666671, + "percentile999": 0.31378533333333636, + "range": 0.312000000000008, + "samples": 300, + "stdev": 0.055205162427605946, + "sum": 30.630002722715243 + }, + "user": { + "iqr": 0.1319999999999974, + "max": 1.2353333333333314, + "mean": 0.20326461725748832, + "median": 0.20200000000000007, + "min": 0.020666666666670608, + "non_zero_mean": 0.20326461725748832, + "non_zero_median": 0.20200000000000007, + "percentile25": 0.12866666666666904, + "percentile75": 0.26066666666666644, + "percentile90": 0.3268000000000026, + "percentile99": 0.49281999999999904, + "percentile999": 1.0393886666666783, + "range": 1.2146666666666608, + "samples": 300, + "stdev": 0.11262712564909984, + "sum": 60.979385177246506 + } + }, + "entropy_available_bits": { + "iqr": 1.6, + "max": 212.53333333333333, + "mean": 11.37666668154963, + "median": 0.7666666666666666, + "min": 0.0, + "non_zero_mean": 20.19526629860881, + "non_zero_median": 1.4, + "percentile25": 0.0, + "percentile75": 1.6, + "percentile90": 2.8200000000000043, + "percentile99": 211.93466666666666, + "percentile999": 212.3938, + "range": 212.53333333333333, + "samples": 300, + "stdev": 46.00087670409667, + "sum": 3413.0000044648905 + }, + "memory": { + "used": { + "iqr": 20170752.0, + "max": 1110482944.0, + "mean": 439779082.24, + "median": 393068544.0, + "min": 374755328.0, + "non_zero_mean": 439779082.24, + "non_zero_median": 393068544.0, + "percentile25": 387190784.0, + "percentile75": 407361536.0, + "percentile90": 531688652.80000037, + "percentile99": 1060299243.5199997, + "percentile999": 1110081241.088, + "range": 735727616.0, + "samples": 300, + "stdev": 137114273.5911923, + "sum": 131933724672.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 4505.6, + "max": 59575559.4, + "mean": 983061.0030120082, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2681075.4627600224, + "non_zero_median": 10103.466666666667, + "percentile25": 0.0, + "percentile75": 4505.6, + "percentile90": 25941.333333333394, + "percentile99": 57682300.81866667, + "percentile999": 59478072.961600006, + "range": 59575559.4, + "samples": 300, + "stdev": 7487676.666024999, + "sum": 294918300.9036027 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 400.450628, + "mean": 3.834116266666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 30.26933894736842, + "non_zero_median": 0.024302, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0022202000000000033, + "percentile99": 180.91000454999985, + "percentile999": 355.6042934410033, + "range": 400.450628, + "samples": 300, + "stdev": 31.48844379277046, + "sum": 1150.2348799999997 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.6810515, + "max": 475.441593, + "mean": 27.301021343333332, + "median": 0.0, + "min": -0.000839, + "non_zero_mean": 61.58125115037594, + "non_zero_median": 1.189577, + "percentile25": 0.0, + "percentile75": 0.6810515, + "percentile90": 85.95422920000013, + "percentile99": 430.58158523999987, + "percentile999": 470.95367804900036, + "range": 475.442432, + "samples": 300, + "stdev": 85.12451523463557, + "sum": 8190.306402999997 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 6.466666666666667, + "mean": 0.15688888888888888, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.137777777777778, + "non_zero_median": 2.933333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.935333333333331, + "percentile999": 6.008200000000033, + "range": 6.466666666666667, + "samples": 300, + "stdev": 0.7477208940271739, + "sum": 47.06666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9995.466666666667, + "mean": 419.1766666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8383.533333333333, + "non_zero_median": 8966.466666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9989.559333333333, + "percentile999": 9995.406866666666, + "range": 9995.466666666667, + "samples": 300, + "stdev": 1868.9569553768897, + "sum": 125753.0 + }, + "pg_stat_database_blks_hit": { + "iqr": 19233.966666666667, + "max": 198740.26666666666, + "mean": 23185.76535667293, + "median": 23071.13333333333, + "min": 2446.4, + "non_zero_mean": 23185.76535667293, + "non_zero_median": 23071.13333333333, + "percentile25": 11371.666666666668, + "percentile75": 30605.633333333335, + "percentile90": 39950.68, + "percentile99": 52044.78333333313, + "percentile999": 186345.46086666756, + "range": 196293.86666666667, + "samples": 300, + "stdev": 18298.391841296438, + "sum": 6955729.607001874 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5333333333333333, + "max": 214.86666666666667, + "mean": 1.2315570274294647, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.5589329461132464, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.6066666666666682, + "percentile99": 3.6746666666666594, + "percentile999": 152.53513333333788, + "range": 214.86666666666667, + "samples": 300, + "stdev": 12.402208284460032, + "sum": 369.46710822883887 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 300, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5333333333333333, + "max": 214.86666666666667, + "mean": 1.2315570274294647, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.5589329461132464, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.6066666666666682, + "percentile99": 3.6746666666666594, + "percentile999": 152.53513333333788, + "range": 214.86666666666667, + "samples": 300, + "stdev": 12.402208284460032, + "sum": 369.46710822883887 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.2333722248149876, + "max": 4001.733333333333, + "mean": 36.20680292577475, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 89.7689328738217, + "non_zero_median": 8.0, + "percentile25": 0.0, + "percentile75": 2.2333722248149876, + "percentile90": 55.6, + "percentile99": 88.28066666666665, + "percentile999": 4001.3346666666666, + "range": 4001.733333333333, + "samples": 300, + "stdev": 325.9997628885316, + "sum": 10862.040877732428 + }, + "pg_stat_database_tup_fetched": { + "iqr": 9229.783333333333, + "max": 393466.86666666664, + "mean": 13042.250524338724, + "median": 12250.9, + "min": 1348.8, + "non_zero_mean": 13042.250524338724, + "non_zero_median": 12250.9, + "percentile25": 5785.7, + "percentile75": 15015.483333333334, + "percentile90": 21512.260000000013, + "percentile99": 30314.80133333333, + "percentile999": 287808.2998000077, + "range": 392118.06666666665, + "samples": 300, + "stdev": 23252.306259048382, + "sum": 3912675.157301616 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.0, + "max": 8002.066666666667, + "mean": 40.519563817347766, + "median": 2.1333333333333333, + "min": 0.0, + "non_zero_mean": 48.429757550614866, + "non_zero_median": 3.066666666666667, + "percentile25": 1.0, + "percentile75": 9.0, + "percentile90": 52.96666666666676, + "percentile99": 128.2626666666666, + "percentile999": 5653.640933333505, + "range": 8002.066666666667, + "samples": 300, + "stdev": 462.12677953362413, + "sum": 12155.869145204324 + }, + "pg_stat_database_tup_returned": { + "iqr": 31093.15, + "max": 501399.5333333333, + "mean": 41743.146962281986, + "median": 33984.7, + "min": 3355.6, + "non_zero_mean": 41743.146962281986, + "non_zero_median": 33984.7, + "percentile25": 17343.6, + "percentile75": 48436.75, + "percentile90": 88214.99333333338, + "percentile99": 168921.3753333333, + "percentile999": 477119.1785333351, + "range": 498043.93333333335, + "samples": 300, + "stdev": 46947.598271962655, + "sum": 12522944.088684596 + }, + "pg_stat_database_tup_updated": { + "iqr": 11.5, + "max": 62.86666666666667, + "mean": 9.200645088050907, + "median": 6.7, + "min": 0.2, + "non_zero_mean": 9.200645088050907, + "non_zero_median": 6.7, + "percentile25": 3.1833333333333336, + "percentile75": 14.683333333333334, + "percentile90": 21.473333333333333, + "percentile99": 33.67533333333326, + "percentile999": 57.205600000000416, + "range": 62.666666666666664, + "samples": 300, + "stdev": 8.663070880630178, + "sum": 2760.193526415273 + }, + "pg_stat_database_xact_commit": { + "iqr": 112.86666666666665, + "max": 647.7333333333333, + "mean": 110.27769118506718, + "median": 81.46666666666667, + "min": 5.266666666666667, + "non_zero_mean": 110.27769118506718, + "non_zero_median": 81.46666666666667, + "percentile25": 37.21666666666667, + "percentile75": 150.08333333333331, + "percentile90": 255.12666666666672, + "percentile99": 529.3773333333328, + "percentile999": 644.7234000000002, + "range": 642.4666666666667, + "samples": 300, + "stdev": 112.11367138271467, + "sum": 33083.30735552018 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.5333333333333333, + "mean": 0.11911111511312242, + "median": 0.06666666666666667, + "min": 0.06646726487205051, + "non_zero_mean": 0.11911111511312242, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4, + "percentile99": 0.5333333333333333, + "percentile999": 0.5333333333333333, + "range": 0.4668660684612828, + "samples": 300, + "stdev": 0.13234565378184185, + "sum": 35.73333453393686 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 300, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 24.216666666666665, + "max": 134.6, + "mean": 36.35780213192952, + "median": 30.5, + "min": 2.2666666666666666, + "non_zero_mean": 36.35780213192952, + "non_zero_median": 30.5, + "percentile25": 22.416666666666668, + "percentile75": 46.63333333333333, + "percentile90": 66.14, + "percentile99": 106.94266666666661, + "percentile999": 129.4771333333337, + "range": 132.33333333333331, + "samples": 300, + "stdev": 21.49762734459809, + "sum": 10907.340639578842 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-17T15:04:16.439328+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-17T11_38_32_937_0000", + "ended": "2021-12-17T16:19:09.232887+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.015991242750715173, + "iterations": 1, + "max": 0.6783046440014004, + "mean": 0.5955501117334886, + "median": 0.5928256170009263, + "min": 0.5670267179993971, + "rounds": 15, + "stddev": 0.025979938083651517 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.02435814574982942, + "iterations": 1, + "max": 1.5673157340006583, + "mean": 0.6556866462001684, + "median": 0.5932303959998535, + "min": 0.5688587410004402, + "rounds": 15, + "stddev": 0.25251952267834976 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.01430284100024437, + "iterations": 1, + "max": 0.47483832299985806, + "mean": 0.44756719329980116, + "median": 0.4442330549991311, + "min": 0.4308817969995289, + "rounds": 10, + "stddev": 0.013211352877763838 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.013317264499164594, + "iterations": 1, + "max": 0.4036199790007231, + "mean": 0.30493150516005696, + "median": 0.30327669399957813, + "min": 0.28495801000099164, + "rounds": 100, + "stddev": 0.014604086340383954 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.02102479250061151, + "iterations": 1, + "max": 0.6435200999985682, + "mean": 0.5378284631900533, + "median": 0.5365590609999344, + "min": 0.499787003000165, + "rounds": 100, + "stddev": 0.0209884674626235 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.0695455969998875, + "iterations": 1, + "max": 1.6397872490015288, + "mean": 1.4950199411005087, + "median": 1.4718746675007424, + "min": 1.3957961489995796, + "rounds": 10, + "stddev": 0.06990137554489044 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.005038384251292882, + "iterations": 1, + "max": 0.09247008800048206, + "mean": 0.08148666315425473, + "median": 0.08077885399870866, + "min": 0.07716065299973707, + "rounds": 13, + "stddev": 0.004022897955743688 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.011250740999457776, + "iterations": 1, + "max": 0.09345949499947892, + "mean": 0.08504197300026135, + "median": 0.08373951550038328, + "min": 0.0782721310006309, + "rounds": 10, + "stddev": 0.005578009801953655 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.007146710999222705, + "iterations": 1, + "max": 0.1381227719994058, + "mean": 0.11681925050015707, + "median": 0.11338128699935623, + "min": 0.103899903000638, + "rounds": 10, + "stddev": 0.011155662709942093 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.0028231000014784513, + "iterations": 1, + "max": 0.1427228369993827, + "mean": 0.12360702959958872, + "median": 0.12153532550019008, + "min": 0.12009288799890783, + "rounds": 10, + "stddev": 0.0068157213091806 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.010001747999922372, + "iterations": 1, + "max": 0.36757577499884064, + "mean": 0.3091078428997207, + "median": 0.30609855400052766, + "min": 0.2933999090000725, + "rounds": 10, + "stddev": 0.02152913970338601 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.004281766001440701, + "iterations": 1, + "max": 0.13198521099911886, + "mean": 0.12408309550010017, + "median": 0.12374066699976538, + "min": 0.11785337000037543, + "rounds": 10, + "stddev": 0.004577537164360444 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.009997800499604637, + "iterations": 1, + "max": 0.26266215500072576, + "mean": 0.10055257909136178, + "median": 0.08433440100088774, + "min": 0.07904604200120957, + "rounds": 11, + "stddev": 0.0540181771290753 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.03182589700008975, + "iterations": 1, + "max": 0.7238939670005493, + "mean": 0.6853264325000055, + "median": 0.6866626434994032, + "min": 0.6624506839998503, + "rounds": 10, + "stddev": 0.018860040029814368 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0018592289998196065, + "iterations": 1, + "max": 0.10544139600096969, + "mean": 0.10275770079970244, + "median": 0.10313400249924598, + "min": 0.098693384001308, + "rounds": 10, + "stddev": 0.0020633618376932053 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.004585450999002205, + "iterations": 1, + "max": 0.12066652700013947, + "mean": 0.11648161109969805, + "median": 0.11651953299951856, + "min": 0.11115799599974707, + "rounds": 10, + "stddev": 0.0033437077943677777 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.008573371998863877, + "iterations": 1, + "max": 0.2662444779998623, + "mean": 0.20752688869961275, + "median": 0.2001417944993591, + "min": 0.19466369799920358, + "rounds": 10, + "stddev": 0.021588327393583077 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.007209009751477424, + "iterations": 1, + "max": 0.09540847599964764, + "mean": 0.08654467163614754, + "median": 0.08565799300049548, + "min": 0.07928365499901702, + "rounds": 11, + "stddev": 0.004932639309481925 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.0035435995005173027, + "iterations": 1, + "max": 0.08694797099997231, + "mean": 0.08439169250020011, + "median": 0.08526827550031157, + "min": 0.08163301800050249, + "rounds": 12, + "stddev": 0.001989151906320026 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.035904047999792965, + "iterations": 1, + "max": 0.5805275090006035, + "mean": 0.4525341341002786, + "median": 0.43009154300034425, + "min": 0.41527381200103264, + "rounds": 10, + "stddev": 0.05050206382772114 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.009128226000029827, + "iterations": 1, + "max": 0.49791548199937097, + "mean": 0.43998117030005235, + "median": 0.43536293350007327, + "min": 0.4168697559998691, + "rounds": 10, + "stddev": 0.021631508878902087 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.011883754999871599, + "iterations": 1, + "max": 0.1311217470010888, + "mean": 0.11215453970035014, + "median": 0.10827873200014437, + "min": 0.10257455999999365, + "rounds": 10, + "stddev": 0.010282314634752092 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.004940837750382343, + "iterations": 1, + "max": 0.10137629700147954, + "mean": 0.09577166200010238, + "median": 0.09581990699916787, + "min": 0.09185517900004925, + "rounds": 11, + "stddev": 0.003003726017939529 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.01398552999853564, + "iterations": 1, + "max": 0.2241130130005331, + "mean": 0.20523927449994517, + "median": 0.20277745700059313, + "min": 0.19338461000006646, + "rounds": 10, + "stddev": 0.010144526032873465 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0035888562506443122, + "iterations": 1, + "max": 0.08791170599943143, + "mean": 0.08233348292336674, + "median": 0.08169069799987483, + "min": 0.07878833500035398, + "rounds": 13, + "stddev": 0.0027608214530176964 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.016861236999829998, + "iterations": 1, + "max": 0.25877974099967105, + "mean": 0.23886976190005954, + "median": 0.23690728249948734, + "min": 0.22451137000098242, + "rounds": 10, + "stddev": 0.010247100009532639 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0045083289996910025, + "iterations": 1, + "max": 0.09430598200015083, + "mean": 0.08777228899983432, + "median": 0.08736563949969423, + "min": 0.08313053799975023, + "rounds": 12, + "stddev": 0.003167204562546066 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.007731057999990298, + "iterations": 1, + "max": 0.4010110369999893, + "mean": 0.3876417092002157, + "median": 0.38523187299961137, + "min": 0.3796771159995842, + "rounds": 10, + "stddev": 0.006865217526348761 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.01497699599894986, + "iterations": 1, + "max": 0.2840433279998251, + "mean": 0.2652601753003182, + "median": 0.2647233964999032, + "min": 0.2529919170010544, + "rounds": 10, + "stddev": 0.011152275393675496 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.003333171998747275, + "iterations": 1, + "max": 0.18589624900050694, + "mean": 0.1813945849002266, + "median": 0.18116487100087397, + "min": 0.17728505800005223, + "rounds": 10, + "stddev": 0.0027644692205719967 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.14882767000017338, + "iterations": 1, + "max": 0.27705479100040975, + "mean": 0.20684059460036225, + "median": 0.2597795345009217, + "min": 0.11598380800023733, + "rounds": 10, + "stddev": 0.07759022674037809 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.00595598399922892, + "iterations": 1, + "max": 0.08620795999922848, + "mean": 0.08066028300017933, + "median": 0.07935858300061227, + "min": 0.07693872200070473, + "rounds": 10, + "stddev": 0.00326533538752263 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.008445709001534851, + "iterations": 1, + "max": 0.11760316500112822, + "mean": 0.11047409840011824, + "median": 0.11162437450002471, + "min": 0.10264521799945214, + "rounds": 10, + "stddev": 0.005163169195520164 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.003939437998269568, + "iterations": 1, + "max": 0.08230577500034997, + "mean": 0.07969010819997493, + "median": 0.07966350199876615, + "min": 0.07696400599888875, + "rounds": 10, + "stddev": 0.0020728145662452947 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.005361854999136995, + "iterations": 1, + "max": 0.14676615400094306, + "mean": 0.14195430079998914, + "median": 0.14152536049914488, + "min": 0.13775423699917155, + "rounds": 10, + "stddev": 0.003284928099481815 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.004740952000247489, + "iterations": 1, + "max": 0.09237162200042803, + "mean": 0.08295480966686834, + "median": 0.08230412700049783, + "min": 0.07879944699925545, + "rounds": 12, + "stddev": 0.003838442789545894 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.1630605510017631, + "iterations": 1, + "max": 0.8489068769995356, + "mean": 0.7253029028997844, + "median": 0.6854081764995499, + "min": 0.6294995280004514, + "rounds": 10, + "stddev": 0.08410573891494477 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.015130957999645034, + "iterations": 1, + "max": 0.7200165520007431, + "mean": 0.6963192008000988, + "median": 0.6963998245000766, + "min": 0.6775264099997003, + "rounds": 10, + "stddev": 0.013161567380694189 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.013613975999760441, + "iterations": 1, + "max": 0.33348877300159074, + "mean": 0.3171848697003952, + "median": 0.3171011489994271, + "min": 0.30088286099999095, + "rounds": 10, + "stddev": 0.00941473690345828 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.00373266800033889, + "iterations": 1, + "max": 0.09171496400085744, + "mean": 0.08134447525041348, + "median": 0.08022130049994303, + "min": 0.07742958100061514, + "rounds": 12, + "stddev": 0.00411559645635357 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0017487259997324145, + "iterations": 1, + "max": 0.08127035200050159, + "mean": 0.0789833660003821, + "median": 0.0786857550010609, + "min": 0.07649497800048266, + "rounds": 13, + "stddev": 0.0012978444423168709 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0030871100007061614, + "iterations": 1, + "max": 0.09639613500075939, + "mean": 0.08543923166659322, + "median": 0.08350194949980505, + "min": 0.08172283500061894, + "rounds": 12, + "stddev": 0.004515981148616735 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0017303819995504455, + "iterations": 1, + "max": 0.08435269700021308, + "mean": 0.08165378946166758, + "median": 0.08140999799979909, + "min": 0.07986917100060964, + "rounds": 13, + "stddev": 0.0013469187207027773 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.011411796000174945, + "iterations": 1, + "max": 0.3431652060007764, + "mean": 0.33213194920008393, + "median": 0.33143255850063724, + "min": 0.32496043299943267, + "rounds": 10, + "stddev": 0.00663975302841927 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.008282348999273381, + "iterations": 1, + "max": 0.28257870599918533, + "mean": 0.26174648459982564, + "median": 0.25661736450001627, + "min": 0.2532016390014178, + "rounds": 10, + "stddev": 0.010760902725555646 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.02464309099923412, + "iterations": 1, + "max": 1.2054402520006988, + "mean": 1.0600465486002577, + "median": 1.0310096125003838, + "min": 1.0017919919991982, + "rounds": 10, + "stddev": 0.07566747108779438 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.3249859640000068, + "iterations": 1, + "max": 10.31640362500002, + "mean": 9.92330125089993, + "median": 10.058811203000005, + "min": 9.226259790000768, + "rounds": 10, + "stddev": 0.3615151550703287 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.04606210600104532, + "iterations": 1, + "max": 1.2049822910012153, + "mean": 1.1611914507006076, + "median": 1.1618135860007897, + "min": 1.092773013000624, + "rounds": 10, + "stddev": 0.03507631706016511 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.2900819189999311, + "iterations": 1, + "max": 11.136565369999516, + "mean": 10.690723728400007, + "median": 10.656674728500548, + "min": 10.33191754299878, + "rounds": 10, + "stddev": 0.25608839224344493 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.44715761699990253, + "iterations": 1, + "max": 1.8834302110008139, + "mean": 1.5025021911003933, + "median": 1.3902069565001511, + "min": 1.1625270140011708, + "rounds": 10, + "stddev": 0.2706123493182868 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.018317131000003428, + "iterations": 1, + "max": 0.4665995129998919, + "mean": 0.45529756449977865, + "median": 0.45931424249920383, + "min": 0.434863888000109, + "rounds": 10, + "stddev": 0.011742721318838599 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.2591150810003455, + "iterations": 1, + "max": 13.195791874999486, + "mean": 12.831760750799912, + "median": 12.858209032999184, + "min": 12.550116803000492, + "rounds": 10, + "stddev": 0.20754933780013224 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.016101664001325844, + "iterations": 1, + "max": 0.47263463899980707, + "mean": 0.4604167616998893, + "median": 0.458204440499685, + "min": 0.44840203300009307, + "rounds": 10, + "stddev": 0.008552666215083984 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.27985497299960116, + "iterations": 1, + "max": 1.0924659330012219, + "mean": 0.6691182276004838, + "median": 0.6379229395006405, + "min": 0.4753146740004013, + "rounds": 10, + "stddev": 0.19018448172265398 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.2663676940010191, + "iterations": 1, + "max": 13.916649390999737, + "mean": 13.410965879000106, + "median": 13.325551344501037, + "min": 13.163264378999884, + "rounds": 10, + "stddev": 0.25098952775076366 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.2760147800017876, + "iterations": 1, + "max": 9.630364185999497, + "mean": 9.198260041099639, + "median": 9.133879040499778, + "min": 8.923429317999762, + "rounds": 10, + "stddev": 0.2175293921861553 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 305.01030279099905, + "mean": 305.01030279099905, + "median": 305.01030279099905, + "min": 305.01030279099905, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 468.1269327439986, + "mean": 468.1269327439986, + "median": 468.1269327439986, + "min": 468.1269327439986, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.010215066000455408, + "iterations": 1, + "max": 0.30519323800035636, + "mean": 0.2891550814996663, + "median": 0.2876007259992548, + "min": 0.27874235099989164, + "rounds": 10, + "stddev": 0.008133202943696054 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.029217857001640368, + "iterations": 1, + "max": 0.8678937289987516, + "mean": 0.7385346019000281, + "median": 0.7243114409993723, + "min": 0.6902021770001738, + "rounds": 10, + "stddev": 0.048556640436976314 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.1446320340019156, + "iterations": 1, + "max": 11.207822730000771, + "mean": 11.016585795900028, + "median": 10.990524281000035, + "min": 10.918538463000004, + "rounds": 10, + "stddev": 0.09491091678615582 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.38227385000027425, + "iterations": 1, + "max": 3.774736766999922, + "mean": 3.4949984989996663, + "median": 3.5253716934994372, + "min": 2.9048414139997476, + "rounds": 10, + "stddev": 0.2729921929362503 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.5708625479983311, + "iterations": 1, + "max": 4.990508647999377, + "mean": 2.5223266093998973, + "median": 2.3052338904990393, + "min": 1.6827341950011032, + "rounds": 10, + "stddev": 0.9397013307966712 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 3.0673459870013176, + "iterations": 1, + "max": 40.467304718999, + "mean": 30.44127917390033, + "median": 29.03317671900004, + "min": 28.13081403400065, + "rounds": 10, + "stddev": 3.7438404385705253 + } + }, + "started": "2021-12-17T14:49:44.312570+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-chatty.json new file mode 100644 index 00000000..0182f3c7 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-17T13:57:53.925301+00:00", + "golden": "true", + "id": "run-2021-12-17T11_38_32_937_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 12152.133333333335, + "max": 51558.26666666666, + "mean": 29528.88862179798, + "median": 34750.2, + "min": 2698.2666666666664, + "non_zero_mean": 29528.88862179798, + "non_zero_median": 34750.2, + "percentile25": 23684.600000000002, + "percentile75": 35836.73333333334, + "percentile90": 36641.573333333334, + "percentile99": 42537.38666666671, + "percentile999": 50656.17866666682, + "range": 48860.0, + "samples": 67, + "stdev": 10462.395315289265, + "sum": 1978435.5376604642 + } + }, + "cpu": { + "system": { + "iqr": 0.27533333333333443, + "max": 1.2820000000000014, + "mean": 1.0542586749541178, + "median": 1.2327478330444028, + "min": 0.1779999999999992, + "non_zero_mean": 1.0542586749541178, + "non_zero_median": 1.2327478330444028, + "percentile25": 0.9683333333333314, + "percentile75": 1.2436666666666658, + "percentile90": 1.2531999999999992, + "percentile99": 1.269239999999998, + "percentile999": 1.2807240000000013, + "range": 1.104000000000002, + "samples": 67, + "stdev": 0.3149186255670336, + "sum": 70.6353312219259 + }, + "user": { + "iqr": 2.244333333333336, + "max": 6.395333333333346, + "mean": 5.109283555898905, + "median": 6.1833333333333265, + "min": 0.5146666666666646, + "non_zero_mean": 5.109283555898905, + "non_zero_median": 6.1833333333333265, + "percentile25": 3.988333333333336, + "percentile75": 6.232666666666672, + "percentile90": 6.281599999999996, + "percentile99": 6.37201333333334, + "percentile999": 6.393001333333346, + "range": 5.880666666666681, + "samples": 67, + "stdev": 1.7329566187546401, + "sum": 342.3219982452268 + } + }, + "entropy_available_bits": { + "iqr": 7.333333333333333, + "max": 215.79999999999998, + "mean": 24.13134328358209, + "median": 3.6, + "min": 0.0, + "non_zero_mean": 35.14782608695652, + "non_zero_median": 6.566666666666666, + "percentile25": 0.0, + "percentile75": 7.333333333333333, + "percentile90": 67.75999999999979, + "percentile99": 215.75599999999997, + "percentile999": 215.79559999999998, + "range": 215.79999999999998, + "samples": 67, + "stdev": 61.27696572094795, + "sum": 1616.8 + }, + "memory": { + "used": { + "iqr": 1343801344.0, + "max": 7545856000.0, + "mean": 6501499246.80597, + "median": 6956687360.0, + "min": 3531898880.0, + "non_zero_mean": 6501499246.80597, + "non_zero_median": 6956687360.0, + "percentile25": 5852633088.0, + "percentile75": 7196434432.0, + "percentile90": 7313330995.2, + "percentile99": 7521441955.84, + "percentile999": 7543414595.584001, + "range": 4013957120.0, + "samples": 67, + "stdev": 920792508.208201, + "sum": 435600449536.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.0666666666666667, + "mean": 0.03781094527363184, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6333333333333333, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.0666666666666667, + "percentile999": 1.0666666666666667, + "range": 1.0666666666666667, + "samples": 67, + "stdev": 0.18540115051528827, + "sum": 2.533333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 2278.033333333333, + "max": 4777.6, + "mean": 3410.0021937957713, + "median": 4382.2, + "min": 48.4, + "non_zero_mean": 3410.0021937957713, + "non_zero_median": 4382.2, + "percentile25": 2222.633333333333, + "percentile75": 4500.666666666666, + "percentile90": 4565.505871894031, + "percentile99": 4705.836, + "percentile999": 4770.423600000002, + "range": 4729.200000000001, + "samples": 67, + "stdev": 1496.6432314479785, + "sum": 228470.1469843167 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 226.29999999999995, + "max": 2054.4666666666667, + "mean": 1538.5094527363185, + "median": 1611.2, + "min": 584.4666666666667, + "non_zero_mean": 1538.5094527363185, + "non_zero_median": 1611.2, + "percentile25": 1468.2666666666667, + "percentile75": 1694.5666666666666, + "percentile90": 1787.8666666666666, + "percentile99": 1970.6026666666671, + "percentile999": 2046.0802666666682, + "range": 1470.0, + "samples": 67, + "stdev": 271.595741550717, + "sum": 103080.13333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.02766666666666806, + "max": 0.2400000000000015, + "mean": 0.15349253731343282, + "median": 0.1580000000000003, + "min": 0.0313333333333361, + "non_zero_mean": 0.15349253731343282, + "non_zero_median": 0.1580000000000003, + "percentile25": 0.14733333333333337, + "percentile75": 0.17500000000000143, + "percentile90": 0.19319999999999934, + "percentile99": 0.23779999999999976, + "percentile999": 0.23978000000000135, + "range": 0.2086666666666654, + "samples": 67, + "stdev": 0.04209578456502044, + "sum": 10.283999999999999 + }, + "user": { + "iqr": 0.04333333333333397, + "max": 0.5713333333333339, + "mean": 0.22979104477611942, + "median": 0.17600000000000005, + "min": 0.046666666666666856, + "non_zero_mean": 0.22979104477611942, + "non_zero_median": 0.17600000000000005, + "percentile25": 0.16166666666666646, + "percentile75": 0.20500000000000043, + "percentile90": 0.5086666666666669, + "percentile99": 0.5559333333333336, + "percentile999": 0.5697933333333342, + "range": 0.5246666666666671, + "samples": 67, + "stdev": 0.13858667122228752, + "sum": 15.395999999999999 + } + }, + "entropy_available_bits": { + "iqr": 0.20000000000000018, + "max": 204.13333333333333, + "mean": 12.623880597014924, + "median": 1.2666666666666666, + "min": 0.3333333333333333, + "non_zero_mean": 12.623880597014924, + "non_zero_median": 1.2666666666666666, + "percentile25": 1.1666666666666665, + "percentile75": 1.3666666666666667, + "percentile90": 1.92, + "percentile99": 198.89733333333336, + "percentile999": 203.60973333333342, + "range": 203.79999999999998, + "samples": 67, + "stdev": 45.43858948451667, + "sum": 845.8 + }, + "memory": { + "used": { + "iqr": 35497984.0, + "max": 531607552.0, + "mean": 454583005.6119403, + "median": 468492288.0, + "min": 377032704.0, + "non_zero_mean": 454583005.6119403, + "non_zero_median": 468492288.0, + "percentile25": 438796288.0, + "percentile75": 474294272.0, + "percentile90": 479681740.8, + "percentile99": 501194752.0000002, + "percentile999": 528566272.00000054, + "range": 154574848.0, + "samples": 67, + "stdev": 32267845.867739484, + "sum": 30457061376.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1737250.1333333333, + "max": 3418248.533333333, + "mean": 761701.1263681592, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1962845.2102564103, + "non_zero_median": 2074487.4666666668, + "percentile25": 0.0, + "percentile75": 1737250.1333333333, + "percentile90": 2871350.6133333333, + "percentile99": 3224687.9573333343, + "percentile999": 3398892.4757333365, + "range": 3418248.533333333, + "samples": 67, + "stdev": 1137453.1421395545, + "sum": 51033975.46666667 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 5.913605, + "mean": 0.09214822388059701, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6859923333333333, + "non_zero_median": 0.01224, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0023045999999999995, + "percentile99": 2.10709790000002, + "percentile999": 5.532954290000067, + "range": 5.913605, + "samples": 67, + "stdev": 0.7222360554047623, + "sum": 6.173930999999999 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0024879999999999998, + "max": 0.626459, + "mean": 0.03662631343283582, + "median": 0.0, + "min": -0.001313, + "non_zero_mean": 0.10669404347826086, + "non_zero_median": 0.019206, + "percentile25": 0.0, + "percentile75": 0.0024879999999999998, + "percentile90": 0.05670839999999996, + "percentile99": 0.47190020000000077, + "percentile999": 0.6110031200000028, + "range": 0.627772, + "samples": 67, + "stdev": 0.11189196643180434, + "sum": 2.4539630000000003 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 12.8, + "mean": 0.45970149253731346, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 10.266666666666667, + "non_zero_median": 9.333333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 10.512000000000013, + "percentile999": 12.571200000000042, + "range": 12.8, + "samples": 67, + "stdev": 2.1738926691887577, + "sum": 30.8 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9987.933333333332, + "mean": 447.1681592039801, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9986.755555555555, + "non_zero_median": 9986.333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9986.877333333334, + "percentile999": 9987.827733333332, + "range": 9987.933333333332, + "samples": 67, + "stdev": 2080.9692687032407, + "sum": 29960.26666666667 + }, + "pg_stat_database_blks_hit": { + "iqr": 3174.2000000000044, + "max": 39836.8, + "mean": 20222.110447761195, + "median": 20538.333333333332, + "min": 8177.066666666667, + "non_zero_mean": 20222.110447761195, + "non_zero_median": 20538.333333333332, + "percentile25": 18861.166666666664, + "percentile75": 22035.36666666667, + "percentile90": 23756.613333333335, + "percentile99": 38733.72000000001, + "percentile999": 39726.49200000002, + "range": 31659.733333333337, + "samples": 67, + "stdev": 5411.593958560664, + "sum": 1354881.4 + }, + "pg_stat_database_blks_read": { + "iqr": 60.266666666666666, + "max": 181.33333333333334, + "mean": 94.77313432835821, + "median": 119.6, + "min": 0.0, + "non_zero_mean": 97.68923076923078, + "non_zero_median": 119.73333333333333, + "percentile25": 65.93333333333334, + "percentile75": 126.2, + "percentile90": 131.04, + "percentile99": 158.14533333333344, + "percentile999": 179.01453333333376, + "range": 181.33333333333334, + "samples": 67, + "stdev": 46.01447446067875, + "sum": 6349.8 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 67, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 60.266666666666666, + "max": 181.33333333333334, + "mean": 94.77313432835821, + "median": 119.6, + "min": 0.0, + "non_zero_mean": 97.68923076923078, + "non_zero_median": 119.73333333333333, + "percentile25": 65.93333333333334, + "percentile75": 126.2, + "percentile90": 131.04, + "percentile99": 158.14533333333344, + "percentile999": 179.01453333333376, + "range": 181.33333333333334, + "samples": 67, + "stdev": 46.01447446067875, + "sum": 6349.8 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.06666666666666667, + "max": 1.4666666666666666, + "mean": 0.22686567164179103, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8941176470588235, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.06666666666666667, + "percentile90": 1.0533333333333332, + "percentile99": 1.4226666666666667, + "percentile999": 1.4622666666666673, + "range": 1.4666666666666666, + "samples": 67, + "stdev": 0.45082187595255263, + "sum": 15.200000000000001 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2626.7999999999993, + "max": 24282.133333333335, + "mean": 6519.0, + "median": 4708.733333333334, + "min": 2867.3333333333335, + "non_zero_mean": 6519.0, + "non_zero_median": 4708.733333333334, + "percentile25": 4197.0, + "percentile75": 6823.799999999999, + "percentile90": 11285.813333333334, + "percentile99": 23691.433333333338, + "percentile999": 24223.063333333346, + "range": 21414.800000000003, + "samples": 67, + "stdev": 4277.762817190159, + "sum": 436773.00000000006 + }, + "pg_stat_database_tup_inserted": { + "iqr": 320.4333333333334, + "max": 1003.7333333333333, + "mean": 504.2686567164179, + "median": 624.2, + "min": 0.0, + "non_zero_mean": 511.90909090909093, + "non_zero_median": 629.3333333333334, + "percentile25": 348.66666666666663, + "percentile75": 669.1, + "percentile90": 696.96, + "percentile99": 859.237333333334, + "percentile999": 989.2837333333359, + "range": 1003.7333333333333, + "samples": 67, + "stdev": 245.35394757765624, + "sum": 33786.0 + }, + "pg_stat_database_tup_returned": { + "iqr": 3269.6333333333323, + "max": 32259.533333333333, + "mean": 8386.845771144279, + "median": 6005.866666666667, + "min": 3707.0, + "non_zero_mean": 8386.845771144279, + "non_zero_median": 6005.866666666667, + "percentile25": 5466.2, + "percentile75": 8735.833333333332, + "percentile90": 15934.93333333333, + "percentile99": 29407.893333333348, + "percentile999": 31974.369333333383, + "range": 28552.533333333333, + "samples": 67, + "stdev": 5555.762148448145, + "sum": 561918.6666666666 + }, + "pg_stat_database_tup_updated": { + "iqr": 4.7666666666666675, + "max": 140.53333333333333, + "mean": 9.791044776119403, + "median": 0.3333333333333333, + "min": 0.0, + "non_zero_mean": 11.714285714285714, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 4.833333333333334, + "percentile90": 24.55999999999999, + "percentile99": 138.42133333333334, + "percentile999": 140.32213333333337, + "range": 140.53333333333333, + "samples": 67, + "stdev": 27.01105371817209, + "sum": 656.0 + }, + "pg_stat_database_xact_commit": { + "iqr": 12.600000000000001, + "max": 142.46666666666667, + "mean": 53.440796019900496, + "median": 44.8, + "min": 7.466666666666667, + "non_zero_mean": 53.440796019900496, + "non_zero_median": 44.8, + "percentile25": 41.03333333333333, + "percentile75": 53.63333333333333, + "percentile90": 90.67999999999996, + "percentile99": 134.2826666666667, + "percentile999": 141.6482666666668, + "range": 135.0, + "samples": 67, + "stdev": 26.639281279735368, + "sum": 3580.5333333333333 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 67, + "stdev": 0.0, + "sum": 4.46666666666667 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003980099502487562, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.09066666666666758, + "percentile999": 0.2490666666666698, + "range": 0.26666666666666666, + "samples": 67, + "stdev": 0.032578518495014724, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 20.699999999999996, + "max": 121.0, + "mean": 67.08955223880596, + "median": 68.73333333333333, + "min": 12.0, + "non_zero_mean": 67.08955223880596, + "non_zero_median": 68.73333333333333, + "percentile25": 57.53333333333334, + "percentile75": 78.23333333333333, + "percentile90": 95.25333333333333, + "percentile99": 109.03200000000007, + "percentile999": 119.80320000000022, + "range": 109.0, + "samples": 67, + "stdev": 20.95566765902555, + "sum": 4494.999999999998 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,8,10,9,11,15,18,20,19,22,23,21,25,24,26,27,29,30,31,33,32,34,28,35,36,37,40,39,42,41,38,43,45,44,46,47,48,49,50,53,52,51,54,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-17T13:38:23.606698Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2021-12-17T13:38:24.713489Z 2021-12-17T13:38:24.525352Z 2021-12-17T13:38:24.216404Z 2021-12-17T13:38:23.713067Z 2021-12-17T13:38:23.713048Z 2021-12-17T13:38:23.674048Z 2021-12-17T13:38:23.606698Z 2021-12-17T13:38:26.599047Z 2021-12-17T13:38:24.778411Z 2021-12-17T13:38:24.730204Z 2021-12-17T13:42:52.462269Z 2021-12-17T13:42:51.962090Z 2021-12-17T13:42:51.414543Z 2021-12-17T13:42:51.399178Z 2021-12-17T13:42:51.401294Z 2021-12-17T13:42:51.361656Z 2021-12-17T13:42:51.346169Z 2021-12-17T13:42:51.321559Z 2021-12-17T13:42:51.317719Z 2021-12-17T13:42:51.243589Z 2021-12-17T13:45:59.256032Z 2021-12-17T13:45:59.222684Z 2021-12-17T13:45:58.713599Z 2021-12-17T13:45:58.702793Z 2021-12-17T13:45:58.629101Z 2021-12-17T13:45:58.622312Z 2021-12-17T13:45:58.620718Z 2021-12-17T13:45:58.608538Z 2021-12-17T13:45:58.604195Z 2021-12-17T13:45:58.604121Z 2021-12-17T13:49:06.537214Z 2021-12-17T13:49:05.565297Z 2021-12-17T13:49:05.014570Z 2021-12-17T13:49:04.998251Z 2021-12-17T13:49:04.934428Z 2021-12-17T13:49:04.932472Z 2021-12-17T13:49:04.924006Z 2021-12-17T13:49:04.920995Z 2021-12-17T13:49:04.919124Z 2021-12-17T13:49:04.917488Z 2021-12-17T13:52:13.283883Z 2021-12-17T13:52:12.934062Z 2021-12-17T13:52:12.849247Z 2021-12-17T13:52:12.443962Z 2021-12-17T13:52:12.414517Z 2021-12-17T13:52:12.411222Z 2021-12-17T13:52:12.409908Z 2021-12-17T13:52:12.394973Z 2021-12-17T13:52:12.299421Z 2021-12-17T13:52:12.211900Z", + "created_diff": 1.5109298, + "ended": "2021-12-17T13:54:56.679560Z", + "error_job_ids": " []", + "event_first": "2021-12-17T13:38:40.919599Z 2021-12-17T13:40:05.941402Z 2021-12-17T13:38:35.202533Z 2021-12-17T13:39:48.127358Z 2021-12-17T13:38:30.294500Z 2021-12-17T13:39:23.803547Z 2021-12-17T13:38:29.773303Z 2021-12-17T13:39:35.545610Z 2021-12-17T13:38:37.927972Z 2021-12-17T13:40:30.743604Z 2021-12-17T13:43:08.384790Z 2021-12-17T13:43:08.124559Z 2021-12-17T13:43:00.756492Z 2021-12-17T13:43:00.229485Z 2021-12-17T13:43:00.872447Z 2021-12-17T13:43:01.032323Z 2021-12-17T13:43:02.036625Z 2021-12-17T13:42:59.977767Z 2021-12-17T13:42:58.732006Z 2021-12-17T13:42:59.207808Z 2021-12-17T13:46:13.402761Z 2021-12-17T13:46:14.536928Z 2021-12-17T13:46:10.253395Z 2021-12-17T13:46:12.403430Z 2021-12-17T13:46:10.006499Z 2021-12-17T13:46:06.817330Z 2021-12-17T13:46:06.605907Z 2021-12-17T13:46:05.815839Z 2021-12-17T13:46:05.821195Z 2021-12-17T13:46:05.840556Z 2021-12-17T13:49:23.468884Z 2021-12-17T13:49:24.249081Z 2021-12-17T13:49:14.633473Z 2021-12-17T13:49:14.795311Z 2021-12-17T13:49:14.521254Z 2021-12-17T13:49:13.742518Z 2021-12-17T13:49:12.657429Z 2021-12-17T13:49:12.233359Z 2021-12-17T13:49:12.222615Z 2021-12-17T13:49:11.980676Z 2021-12-17T13:52:29.332358Z 2021-12-17T13:52:28.079752Z 2021-12-17T13:52:23.558260Z 2021-12-17T13:52:24.130292Z 2021-12-17T13:52:20.820155Z 2021-12-17T13:52:19.934725Z 2021-12-17T13:52:19.299218Z 2021-12-17T13:52:20.437383Z 2021-12-17T13:52:20.318403Z 2021-12-17T13:52:19.438083Z", + "event_last": "2021-12-17T13:41:20.169313Z 2021-12-17T13:42:27.053334Z 2021-12-17T13:41:14.882047Z 2021-12-17T13:42:16.986340Z 2021-12-17T13:41:09.654211Z 2021-12-17T13:41:59.441651Z 2021-12-17T13:41:09.382155Z 2021-12-17T13:42:07.964246Z 2021-12-17T13:41:18.053476Z 2021-12-17T13:42:42.518819Z 2021-12-17T13:45:35.071845Z 2021-12-17T13:45:35.182555Z 2021-12-17T13:45:32.326215Z 2021-12-17T13:45:32.553320Z 2021-12-17T13:45:32.858143Z 2021-12-17T13:45:30.544550Z 2021-12-17T13:45:33.234648Z 2021-12-17T13:45:30.520753Z 2021-12-17T13:45:32.000994Z 2021-12-17T13:45:29.738619Z 2021-12-17T13:48:44.174956Z 2021-12-17T13:48:42.286200Z 2021-12-17T13:48:39.880460Z 2021-12-17T13:48:40.840078Z 2021-12-17T13:48:38.864682Z 2021-12-17T13:48:40.071831Z 2021-12-17T13:48:38.682499Z 2021-12-17T13:48:39.785646Z 2021-12-17T13:48:36.656242Z 2021-12-17T13:48:39.455399Z 2021-12-17T13:51:49.988685Z 2021-12-17T13:51:53.179389Z 2021-12-17T13:51:46.223062Z 2021-12-17T13:51:47.407765Z 2021-12-17T13:51:46.525239Z 2021-12-17T13:51:43.743277Z 2021-12-17T13:51:45.455503Z 2021-12-17T13:51:44.113515Z 2021-12-17T13:51:45.951508Z 2021-12-17T13:51:43.983887Z 2021-12-17T13:54:56.679560Z 2021-12-17T13:54:56.636347Z 2021-12-17T13:54:52.505641Z 2021-12-17T13:54:52.790340Z 2021-12-17T13:54:52.031575Z 2021-12-17T13:54:50.910788Z 2021-12-17T13:54:51.680219Z 2021-12-17T13:54:51.583191Z 2021-12-17T13:54:51.748088Z 2021-12-17T13:54:51.409291Z", + "failed_job_ids": " []", + "finished": "2021-12-17T13:41:23.956464Z 2021-12-17T13:42:30.961686Z 2021-12-17T13:41:18.509653Z 2021-12-17T13:42:20.338963Z 2021-12-17T13:41:13.850224Z 2021-12-17T13:42:04.185915Z 2021-12-17T13:41:13.487680Z 2021-12-17T13:42:11.621631Z 2021-12-17T13:41:22.188571Z 2021-12-17T13:42:44.730489Z 2021-12-17T13:45:38.692605Z 2021-12-17T13:45:38.788866Z 2021-12-17T13:45:33.640164Z 2021-12-17T13:45:34.174419Z 2021-12-17T13:45:37.101007Z 2021-12-17T13:45:34.357958Z 2021-12-17T13:45:36.485395Z 2021-12-17T13:45:34.135022Z 2021-12-17T13:45:33.749994Z 2021-12-17T13:45:32.658280Z 2021-12-17T13:48:45.748849Z 2021-12-17T13:48:45.848335Z 2021-12-17T13:48:43.680593Z 2021-12-17T13:48:44.872394Z 2021-12-17T13:48:42.822596Z 2021-12-17T13:48:44.303659Z 2021-12-17T13:48:42.368441Z 2021-12-17T13:48:43.113417Z 2021-12-17T13:48:38.891677Z 2021-12-17T13:48:42.864246Z 2021-12-17T13:51:53.912050Z 2021-12-17T13:51:55.274749Z 2021-12-17T13:51:49.315713Z 2021-12-17T13:51:50.863416Z 2021-12-17T13:51:50.040299Z 2021-12-17T13:51:46.453927Z 2021-12-17T13:51:47.493499Z 2021-12-17T13:51:47.307604Z 2021-12-17T13:51:49.206331Z 2021-12-17T13:51:46.205686Z 2021-12-17T13:54:58.670615Z 2021-12-17T13:54:58.550374Z 2021-12-17T13:54:55.154237Z 2021-12-17T13:54:57.101145Z 2021-12-17T13:54:54.579809Z 2021-12-17T13:54:53.773527Z 2021-12-17T13:54:52.927315Z 2021-12-17T13:54:54.670571Z 2021-12-17T13:54:53.318142Z 2021-12-17T13:54:54.286760Z", + "finished_diff": 23.8284832, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 258.274604, + "mean": 170.86031228000004, + "min": 159.156395 + }, + "jobs_events_duration": { + "max": 160.125504, + "mean": 151.12782596, + "min": 131.775215 + }, + "jobs_events_lag": { + "max": -1.247096, + "mean": -3.1170572999999995, + "min": -4.744264 + }, + "jobs_waiting": { + "max": 2.840706, + "mean": 1.47636962, + "min": 0.928999 + }, + "modified": "2021-12-17T13:38:25.832575Z 2021-12-17T13:38:25.798483Z 2021-12-17T13:38:25.762934Z 2021-12-17T13:38:24.327652Z 2021-12-17T13:38:24.291846Z 2021-12-17T13:38:24.260356Z 2021-12-17T13:38:24.228339Z 2021-12-17T13:38:27.405397Z 2021-12-17T13:38:25.920712Z 2021-12-17T13:38:25.868388Z 2021-12-17T13:42:53.687921Z 2021-12-17T13:42:53.624763Z 2021-12-17T13:42:52.222189Z 2021-12-17T13:42:52.154587Z 2021-12-17T13:42:52.187455Z 2021-12-17T13:42:52.121528Z 2021-12-17T13:42:52.088467Z 2021-12-17T13:42:52.050074Z 2021-12-17T13:42:52.016542Z 2021-12-17T13:42:51.981902Z 2021-12-17T13:46:00.804346Z 2021-12-17T13:46:00.740927Z 2021-12-17T13:45:59.443241Z 2021-12-17T13:45:59.410767Z 2021-12-17T13:45:59.377269Z 2021-12-17T13:45:59.338245Z 2021-12-17T13:45:59.300855Z 2021-12-17T13:45:59.268713Z 2021-12-17T13:45:59.233401Z 2021-12-17T13:45:59.200114Z 2021-12-17T13:49:07.915958Z 2021-12-17T13:49:07.841255Z 2021-12-17T13:49:05.770572Z 2021-12-17T13:49:05.736582Z 2021-12-17T13:49:05.702095Z 2021-12-17T13:49:05.666907Z 2021-12-17T13:49:05.632245Z 2021-12-17T13:49:05.598787Z 2021-12-17T13:49:05.564655Z 2021-12-17T13:49:05.528660Z 2021-12-17T13:52:14.442965Z 2021-12-17T13:52:14.387795Z 2021-12-17T13:52:14.280064Z 2021-12-17T13:52:13.047205Z 2021-12-17T13:52:13.013940Z 2021-12-17T13:52:12.981743Z 2021-12-17T13:52:12.940921Z 2021-12-17T13:52:12.901009Z 2021-12-17T13:52:12.867927Z 2021-12-17T13:52:12.826156Z", + "modified_diff": 2.0982832, + "started": "2021-12-17T13:38:26.363204Z 2021-12-17T13:38:26.228498Z 2021-12-17T13:38:26.138704Z 2021-12-17T13:38:24.820271Z 2021-12-17T13:38:24.711181Z 2021-12-17T13:38:24.700682Z 2021-12-17T13:38:24.535697Z 2021-12-17T13:38:27.614116Z 2021-12-17T13:38:26.543169Z 2021-12-17T13:38:26.455885Z 2021-12-17T13:42:54.299534Z 2021-12-17T13:42:54.097677Z 2021-12-17T13:42:53.156144Z 2021-12-17T13:42:52.988839Z 2021-12-17T13:42:53.087566Z 2021-12-17T13:42:52.898512Z 2021-12-17T13:42:52.800422Z 2021-12-17T13:42:52.709555Z 2021-12-17T13:42:52.603101Z 2021-12-17T13:42:52.526172Z 2021-12-17T13:46:01.400498Z 2021-12-17T13:46:01.219404Z 2021-12-17T13:46:00.316780Z 2021-12-17T13:46:00.352093Z 2021-12-17T13:46:00.113766Z 2021-12-17T13:45:59.998333Z 2021-12-17T13:45:59.910634Z 2021-12-17T13:45:59.827035Z 2021-12-17T13:45:59.735282Z 2021-12-17T13:45:59.658323Z 2021-12-17T13:49:08.342588Z 2021-12-17T13:49:08.406003Z 2021-12-17T13:49:06.717045Z 2021-12-17T13:49:06.500967Z 2021-12-17T13:49:06.415224Z 2021-12-17T13:49:06.299150Z 2021-12-17T13:49:06.238517Z 2021-12-17T13:49:06.134042Z 2021-12-17T13:49:06.033401Z 2021-12-17T13:49:05.962364Z 2021-12-17T13:52:15.080907Z 2021-12-17T13:52:15.007614Z 2021-12-17T13:52:14.781570Z 2021-12-17T13:52:13.805800Z 2021-12-17T13:52:13.667534Z 2021-12-17T13:52:13.567752Z 2021-12-17T13:52:13.501168Z 2021-12-17T13:52:13.401137Z 2021-12-17T13:52:13.313589Z 2021-12-17T13:52:13.231899Z", + "started_diff": 2.1773206, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 15, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-17T13:57:51.123512+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-launching.json new file mode 100644 index 00000000..8739b23e --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-17T11_38_32_937_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-17T14:48:50.952560+00:00", + "golden": "true", + "id": "run-2021-12-17T11_38_32_937_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17075.133333333335, + "max": 30501.4, + "mean": 10346.402048038824, + "median": 6791.466666666667, + "min": 1374.0, + "non_zero_mean": 10346.402048038824, + "non_zero_median": 6791.466666666667, + "percentile25": 1863.1999999999998, + "percentile75": 18938.333333333336, + "percentile90": 22825.653333333335, + "percentile99": 28726.322666666674, + "percentile999": 30323.89226666668, + "range": 29127.4, + "samples": 89, + "stdev": 9057.494282140851, + "sum": 920829.7822754555 + } + }, + "cpu": { + "system": { + "iqr": 1.8006666666666757, + "max": 2.450666666666658, + "mean": 0.9516553072572236, + "median": 0.5133333333333325, + "min": 0.03933333333334303, + "non_zero_mean": 0.9516553072572236, + "non_zero_median": 0.5133333333333325, + "percentile25": 0.05599999999999454, + "percentile75": 1.8566666666666702, + "percentile90": 2.1945333333333292, + "percentile99": 2.4154666666666675, + "percentile999": 2.4471466666666593, + "range": 2.4113333333333147, + "samples": 89, + "stdev": 0.903733710388016, + "sum": 84.69732234589289 + }, + "user": { + "iqr": 5.239333333333306, + "max": 6.185333333333332, + "mean": 2.926637030270575, + "median": 1.9893333333333276, + "min": 0.0966666666666697, + "non_zero_mean": 2.926637030270575, + "non_zero_median": 1.9893333333333276, + "percentile25": 0.36466666666666847, + "percentile75": 5.603999999999974, + "percentile90": 5.8931999999999976, + "percentile99": 6.130186666666683, + "percentile999": 6.179818666666667, + "range": 6.088666666666662, + "samples": 89, + "stdev": 2.535625474071764, + "sum": 260.47069569408126 + } + }, + "entropy_available_bits": { + "iqr": 4.2, + "max": 214.35775511564353, + "mean": 25.47967919686507, + "median": 2.066666666666667, + "min": 0.0, + "non_zero_mean": 32.86509345682596, + "non_zero_median": 4.066666666666666, + "percentile25": 0.26666666666666666, + "percentile75": 4.466666666666667, + "percentile90": 205.50666666666666, + "percentile99": 213.7495972805439, + "percentile999": 214.29693933213358, + "range": 214.35775511564353, + "samples": 89, + "stdev": 66.37701307873006, + "sum": 2267.6914485209913 + }, + "memory": { + "used": { + "iqr": 7202103296.0, + "max": 19823751168.0, + "mean": 9483098825.348314, + "median": 6734909440.0, + "min": 5183385600.0, + "non_zero_mean": 9483098825.348314, + "non_zero_median": 6734909440.0, + "percentile25": 6336405504.0, + "percentile75": 13538508800.0, + "percentile90": 16627730022.4, + "percentile99": 19372773048.320004, + "percentile999": 19778653356.032, + "range": 14640365568.0, + "samples": 89, + "stdev": 4300149224.308019, + "sum": 843995795456.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 24.333333333333332, + "mean": 0.5161048689138577, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9.186666666666667, + "non_zero_median": 1.0666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 20.285333333333355, + "percentile999": 23.92853333333336, + "range": 24.333333333333332, + "samples": 89, + "stdev": 3.3016911505354347, + "sum": 45.93333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 928.8000000000001, + "max": 1915.4, + "mean": 460.17700034337963, + "median": 128.06666666666666, + "min": 0.0, + "non_zero_mean": 465.40628443819077, + "non_zero_median": 130.43333333333334, + "percentile25": 3.0, + "percentile75": 931.8000000000001, + "percentile90": 1263.3066666666668, + "percentile99": 1682.7866666666678, + "percentile999": 1892.1386666666683, + "range": 1915.4, + "samples": 89, + "stdev": 546.3342452823384, + "sum": 40955.75303056079 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 1977.6666666666667, + "max": 3430.866666666667, + "mean": 1692.2029962546817, + "median": 1925.7333333333333, + "min": 392.06666666666666, + "non_zero_mean": 1692.2029962546817, + "non_zero_median": 1925.7333333333333, + "percentile25": 660.7333333333333, + "percentile75": 2638.4, + "percentile90": 2983.0533333333333, + "percentile99": 3358.1200000000003, + "percentile999": 3423.5920000000006, + "range": 3038.8, + "samples": 89, + "stdev": 1046.126057882519, + "sum": 150606.06666666662 + } + }, + "cpu": { + "system": { + "iqr": 0.2433333333333337, + "max": 0.4906666666666676, + "mean": 0.16480149812734085, + "median": 0.16266666666667218, + "min": 0.016666666666662874, + "non_zero_mean": 0.16480149812734085, + "non_zero_median": 0.16266666666667218, + "percentile25": 0.03266666666666538, + "percentile75": 0.2759999999999991, + "percentile90": 0.3545333333333346, + "percentile99": 0.4742399999999992, + "percentile999": 0.48902400000000085, + "range": 0.4740000000000047, + "samples": 89, + "stdev": 0.1386939869566525, + "sum": 14.667333333333337 + }, + "user": { + "iqr": 0.3100000000000004, + "max": 0.8613333333333363, + "mean": 0.2190112359550562, + "median": 0.19400000000000167, + "min": 0.01666666666666856, + "non_zero_mean": 0.2190112359550562, + "non_zero_median": 0.19400000000000167, + "percentile25": 0.04533333333333379, + "percentile75": 0.35533333333333417, + "percentile90": 0.4242666666666666, + "percentile99": 0.8443200000000035, + "percentile999": 0.8596320000000032, + "range": 0.8446666666666677, + "samples": 89, + "stdev": 0.20040557645519483, + "sum": 19.492000000000008 + } + }, + "entropy_available_bits": { + "iqr": 3.2, + "max": 212.6, + "mean": 12.782771535580524, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 30.74774774774775, + "non_zero_median": 3.4, + "percentile25": 0.0, + "percentile75": 3.2, + "percentile90": 4.306666666666668, + "percentile99": 210.89866666666666, + "percentile999": 212.42986666666667, + "range": 212.6, + "samples": 89, + "stdev": 48.029785085534314, + "sum": 1137.666666666667 + }, + "memory": { + "used": { + "iqr": 484188160.0, + "max": 1166999552.0, + "mean": 618983930.247191, + "median": 449576960.0, + "min": 371744768.0, + "non_zero_mean": 618983930.247191, + "non_zero_median": 449576960.0, + "percentile25": 382390272.0, + "percentile75": 866578432.0, + "percentile90": 1070761574.4000001, + "percentile99": 1165986693.1200001, + "percentile999": 1166898266.112, + "range": 795254784.0, + "samples": 89, + "stdev": 283369473.6058459, + "sum": 55089569792.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 15291.733333333334, + "max": 55830177.0, + "mean": 641768.6883895131, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2115459.750617284, + "non_zero_median": 47513.6, + "percentile25": 0.0, + "percentile75": 15291.733333333334, + "percentile90": 59746.98666666667, + "percentile99": 6831785.50666692, + "percentile999": 50930337.85066701, + "range": 55830177.0, + "samples": 89, + "stdev": 5916519.445744921, + "sum": 57117413.266666666 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 1.808454, + "mean": 0.029801696629213482, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.22102925, + "non_zero_median": 0.015339, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0020084000000000074, + "percentile99": 0.8030100000000052, + "percentile999": 1.707909600000007, + "range": 1.808454, + "samples": 89, + "stdev": 0.20344022235158843, + "sum": 2.6523510000000003 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.569274, + "max": 11.278288, + "mean": 0.7414861123595505, + "median": 0.007572, + "min": 0.0, + "non_zero_mean": 1.4346144347826086, + "non_zero_median": 0.5653295, + "percentile25": 0.0, + "percentile75": 0.569274, + "percentile90": 1.3716210000000006, + "percentile99": 9.740779280000009, + "percentile999": 11.12453712800001, + "range": 11.278288, + "samples": 89, + "stdev": 1.9099228720685735, + "sum": 65.99226399999999 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.3333333333333335, + "mean": 0.09288389513108615, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.066666666666667, + "non_zero_median": 1.7666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.984000000000007, + "percentile999": 3.1984000000000097, + "range": 3.3333333333333335, + "samples": 89, + "stdev": 0.45909638713412354, + "sum": 8.266666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9976.066666666668, + "mean": 324.11685393258426, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 7211.6, + "non_zero_median": 6357.266666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 6858.696000000016, + "percentile999": 9664.329600000023, + "range": 9976.066666666668, + "samples": 89, + "stdev": 1540.7624583165527, + "sum": 28846.4 + }, + "pg_stat_database_blks_hit": { + "iqr": 18786.86666666667, + "max": 91623.33333333333, + "mean": 19654.505617977527, + "median": 13596.533333333333, + "min": 1417.2, + "non_zero_mean": 19654.505617977527, + "non_zero_median": 13596.533333333333, + "percentile25": 7242.6, + "percentile75": 26029.466666666667, + "percentile90": 37497.613333333335, + "percentile99": 90445.18933333334, + "percentile999": 91505.51893333334, + "range": 90206.13333333333, + "samples": 89, + "stdev": 18917.19902737971, + "sum": 1749250.9999999995 + }, + "pg_stat_database_blks_read": { + "iqr": 2.533333333333333, + "max": 10.733333333333333, + "mean": 1.8404494382022472, + "median": 0.8666666666666667, + "min": 0.0, + "non_zero_mean": 2.73, + "non_zero_median": 1.7666666666666666, + "percentile25": 0.0, + "percentile75": 2.533333333333333, + "percentile90": 5.946666666666667, + "percentile99": 10.498666666666669, + "percentile999": 10.709866666666668, + "range": 10.733333333333333, + "samples": 89, + "stdev": 2.5534017313714097, + "sum": 163.8 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 89, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 2.533333333333333, + "max": 10.733333333333333, + "mean": 1.8404494382022472, + "median": 0.8666666666666667, + "min": 0.0, + "non_zero_mean": 2.73, + "non_zero_median": 1.7666666666666666, + "percentile25": 0.0, + "percentile75": 2.533333333333333, + "percentile90": 5.946666666666667, + "percentile99": 10.498666666666669, + "percentile999": 10.709866666666668, + "range": 10.733333333333333, + "samples": 89, + "stdev": 2.5534017313714097, + "sum": 163.8 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 2.466666666666667, + "mean": 0.250187265917603, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.3916666666666666, + "non_zero_median": 1.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 2.0560000000000023, + "percentile999": 2.425600000000003, + "range": 2.466666666666667, + "samples": 89, + "stdev": 0.5913758054916086, + "sum": 22.266666666666673 + }, + "pg_stat_database_tup_fetched": { + "iqr": 10676.733333333334, + "max": 46026.666666666664, + "mean": 10491.303370786516, + "median": 6958.733333333334, + "min": 708.6, + "non_zero_mean": 10491.303370786516, + "non_zero_median": 6958.733333333334, + "percentile25": 3636.733333333333, + "percentile75": 14313.466666666667, + "percentile90": 20367.400000000005, + "percentile99": 45800.15466666667, + "percentile999": 46004.015466666664, + "range": 45318.066666666666, + "samples": 89, + "stdev": 9778.303737351387, + "sum": 933725.9999999999 + }, + "pg_stat_database_tup_inserted": { + "iqr": 25.4, + "max": 83.8, + "mean": 14.759550561797752, + "median": 5.733333333333333, + "min": 0.0, + "non_zero_mean": 21.18709677419355, + "non_zero_median": 14.9, + "percentile25": 0.0, + "percentile75": 25.4, + "percentile90": 42.20000000000001, + "percentile99": 76.87733333333337, + "percentile999": 83.10773333333339, + "range": 83.8, + "samples": 89, + "stdev": 19.56241918169004, + "sum": 1313.6000000000001 + }, + "pg_stat_database_tup_returned": { + "iqr": 17403.533333333333, + "max": 78099.66666666667, + "mean": 16832.604494382023, + "median": 10924.4, + "min": 1663.2666666666667, + "non_zero_mean": 16832.604494382023, + "non_zero_median": 10924.4, + "percentile25": 6479.933333333333, + "percentile75": 23883.466666666667, + "percentile90": 34158.77333333334, + "percentile99": 72062.39733333337, + "percentile999": 77495.93973333338, + "range": 76436.40000000001, + "samples": 89, + "stdev": 15653.561891863583, + "sum": 1498101.8 + }, + "pg_stat_database_tup_updated": { + "iqr": 24.866666666666667, + "max": 57.6, + "mean": 15.056928838951311, + "median": 8.8, + "min": 0.0, + "non_zero_mean": 16.34227642276423, + "non_zero_median": 12.5, + "percentile25": 0.3333333333333333, + "percentile75": 25.2, + "percentile90": 40.066666666666684, + "percentile99": 56.60266666666667, + "percentile999": 57.500266666666676, + "range": 57.6, + "samples": 89, + "stdev": 16.227715257576765, + "sum": 1340.066666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 229.06666666666666, + "max": 556.5333333333333, + "mean": 142.72883895131085, + "median": 116.86666666666666, + "min": 3.933333333333333, + "non_zero_mean": 142.72883895131085, + "non_zero_median": 116.86666666666666, + "percentile25": 10.266666666666667, + "percentile75": 239.33333333333334, + "percentile90": 306.12, + "percentile99": 523.0933333333335, + "percentile999": 553.1893333333335, + "range": 552.6, + "samples": 89, + "stdev": 142.70445364697662, + "sum": 12702.866666666667 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 89, + "stdev": 0.0, + "sum": 5.933333333333332 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.00299625468164794, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.032000000000001215, + "percentile999": 0.24320000000000164, + "range": 0.26666666666666666, + "samples": 89, + "stdev": 0.028266610133502935, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 91.46666666666667, + "max": 163.86666666666667, + "mean": 53.31086142322098, + "median": 50.266666666666666, + "min": 0.0, + "non_zero_mean": 53.916666666666664, + "non_zero_median": 51.599999999999994, + "percentile25": 3.866666666666667, + "percentile75": 95.33333333333333, + "percentile90": 117.4, + "percentile99": 150.90133333333338, + "percentile999": 162.57013333333342, + "range": 163.86666666666667, + "samples": 89, + "stdev": 48.297952508853804, + "sum": 4744.666666666669 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,60,67,62,65,63,64,66,74,72,70,69,75,73,87,82,68,80,83,89,77,78,85,81,76,90,91,79,88,92,84,95,94,97,98,99,100,101,102,104,103,105,108,109,110,106,113,112,114,111,107,116,117,118,123,120,125,122,127,128,132,134,124,130,135,133,115,129,140,141,137,143,145,144,150,151,138,148,119,146,154,155,153,152,158,160,161,159,163,166,167,165,162,164,169,168,170,171,172,173,174,175,177,180,181,176,188,182,179,189,186,193,178,190,191,183,185,197,192,194,184,198,196,187,195,199,200,201,203,202,204,206,208,205,207,209,213,210,211,212,216,217,215,214,221,220,218,222,224,219,223,225,233,231,237,227,235,239,226,229,234,238,230,243,236,242,244,245,247,248,246,240,259,256,254,251,258,249,253,255,261,232,252,260,264,265,250,262,241,266,263,228,268,267,271,257,270,269,272,273,274,275,277,278,276,280,282,283,281,287,285,286,279,288,290,291,296,289,284,292,297,300,294,293,299,303,304,308,298,301,305,302,295,306,307,312,313,309,310,315,318,311,314,316,317,319,321,322,332,320,323,328,325,327,324,326,329,337,339,330,333,336,331,341,340,338,335,346,344,343,347,345,354,351,356,353,355,352,349,359,348,365,357,334,360,362,366,364,350,342,367,361,368,363,358,369,370,372,371,373,374,375,376,377,385,381,379,382,378,386,380,384,383,387,390,391,389,388,396,395,397,393,398,399,394,403,401,392,405,407,400,402,404,406,408,412,411,410,416,414,419,421,409,420,417,418,424,415,413,425,426,423,427,422,431,428,432,429,430,435,436,438,433,434,443,440,441,437,444,442,446,447,445,454,449,448,451,459,450,457,439,452,466,456,463,461,455,453,462,460,464,467,468,469,470,471,458,473,465,472,475,477,474,476,479,488,481,483,482,485,490,478,487,484,480,486,489,493,491,492,494,495,503,496,499,497,508,500,507,498,505,502,510,504,514,501,506,515,513,509,512,517,511,520,526,525,521,524,523,516,529,528,532,527,519,522,518,540,536,530,533,545,531,535,548,547,537,544,543,539,538,553,549,546,552,555,550,557,560,542,562,551,559,556,558,564,554,561,534,565,563,567,571,566,570,568,572,541,569,573", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-17T13:59:47.424320Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-17T13:59:53.353517Z 2021-12-17T13:59:53.173000Z 2021-12-17T13:59:52.967983Z 2021-12-17T13:59:52.909049Z 2021-12-17T13:59:52.823455Z 2021-12-17T13:59:52.441869Z 2021-12-17T13:59:52.328213Z 2021-12-17T13:59:51.857781Z 2021-12-17T13:59:51.786415Z 2021-12-17T13:59:51.744960Z 2021-12-17T13:59:51.653847Z 2021-12-17T13:59:51.579171Z 2021-12-17T13:59:51.436033Z 2021-12-17T13:59:51.285093Z 2021-12-17T13:59:51.242135Z 2021-12-17T13:59:50.878605Z 2021-12-17T13:59:50.635949Z 2021-12-17T13:59:50.604264Z 2021-12-17T13:59:50.599152Z 2021-12-17T13:59:50.552491Z 2021-12-17T13:59:50.326977Z 2021-12-17T13:59:50.256261Z 2021-12-17T13:59:50.121178Z 2021-12-17T13:59:50.050605Z 2021-12-17T13:59:49.833048Z 2021-12-17T13:59:49.706422Z 2021-12-17T13:59:49.518614Z 2021-12-17T13:59:49.484599Z 2021-12-17T13:59:49.393080Z 2021-12-17T13:59:49.291008Z 2021-12-17T13:59:49.060879Z 2021-12-17T13:59:48.940532Z 2021-12-17T13:59:48.885672Z 2021-12-17T13:59:48.720754Z 2021-12-17T13:59:48.491014Z 2021-12-17T13:59:47.424320Z 2021-12-17T14:00:19.373059Z 2021-12-17T14:00:18.989356Z 2021-12-17T14:00:14.045449Z 2021-12-17T14:00:13.675707Z 2021-12-17T14:00:11.602023Z 2021-12-17T14:00:10.491551Z 2021-12-17T14:00:09.253580Z 2021-12-17T14:00:06.307587Z 2021-12-17T14:00:06.110743Z 2021-12-17T14:00:05.990671Z 2021-12-17T14:00:05.720397Z 2021-12-17T14:00:04.568598Z 2021-12-17T14:00:04.486294Z 2021-12-17T14:00:03.980392Z 2021-12-17T14:00:03.658014Z 2021-12-17T14:00:02.922564Z 2021-12-17T14:00:01.477148Z 2021-12-17T14:00:01.270535Z 2021-12-17T14:00:01.174085Z 2021-12-17T14:00:00.908525Z 2021-12-17T14:00:00.869786Z 2021-12-17T14:00:00.779025Z 2021-12-17T14:00:00.740604Z 2021-12-17T14:00:00.613211Z 2021-12-17T14:00:00.427522Z 2021-12-17T14:00:00.176617Z 2021-12-17T13:59:59.997540Z 2021-12-17T13:59:59.875188Z 2021-12-17T13:59:59.305625Z 2021-12-17T13:59:59.233573Z 2021-12-17T13:59:59.174253Z 2021-12-17T13:59:58.525207Z 2021-12-17T13:59:58.491556Z 2021-12-17T13:59:58.383844Z 2021-12-17T13:59:58.237046Z 2021-12-17T13:59:58.121351Z 2021-12-17T13:59:57.794722Z 2021-12-17T13:59:57.599408Z 2021-12-17T13:59:57.469097Z 2021-12-17T13:59:57.288318Z 2021-12-17T13:59:57.076997Z 2021-12-17T13:59:56.777475Z 2021-12-17T13:59:56.776206Z 2021-12-17T13:59:56.549992Z 2021-12-17T13:59:56.418838Z 2021-12-17T13:59:56.410632Z 2021-12-17T13:59:56.254972Z 2021-12-17T13:59:56.195541Z 2021-12-17T13:59:55.891544Z 2021-12-17T13:59:55.664050Z 2021-12-17T13:59:55.229207Z 2021-12-17T13:59:55.205639Z 2021-12-17T13:59:55.189717Z 2021-12-17T13:59:54.761922Z 2021-12-17T13:59:54.634471Z 2021-12-17T13:59:54.529455Z 2021-12-17T13:59:54.502365Z 2021-12-17T13:59:54.340150Z 2021-12-17T13:59:54.238572Z 2021-12-17T13:59:54.114639Z 2021-12-17T13:59:54.039962Z 2021-12-17T13:59:53.655265Z 2021-12-17T13:59:53.492298Z 2021-12-17T13:59:53.442172Z 2021-12-17T14:05:29.992120Z 2021-12-17T14:05:27.548850Z 2021-12-17T14:05:26.506336Z 2021-12-17T14:05:25.806979Z 2021-12-17T14:05:25.439690Z 2021-12-17T14:05:25.285828Z 2021-12-17T14:05:25.275895Z 2021-12-17T14:05:24.192102Z 2021-12-17T14:05:24.084534Z 2021-12-17T14:05:23.616612Z 2021-12-17T14:05:23.301300Z 2021-12-17T14:05:23.213896Z 2021-12-17T14:05:22.944804Z 2021-12-17T14:05:22.387374Z 2021-12-17T14:05:22.299272Z 2021-12-17T14:05:22.114250Z 2021-12-17T14:05:22.092807Z 2021-12-17T14:05:22.051816Z 2021-12-17T14:05:22.052199Z 2021-12-17T14:05:22.031314Z 2021-12-17T14:05:21.952824Z 2021-12-17T14:05:21.887647Z 2021-12-17T14:05:21.772442Z 2021-12-17T14:05:21.627251Z 2021-12-17T14:05:21.567425Z 2021-12-17T14:05:21.253206Z 2021-12-17T14:05:21.176370Z 2021-12-17T14:05:20.729850Z 2021-12-17T14:05:20.429164Z 2021-12-17T14:05:20.132412Z 2021-12-17T14:05:20.001037Z 2021-12-17T14:05:19.902161Z 2021-12-17T14:05:19.684051Z 2021-12-17T14:05:19.321114Z 2021-12-17T14:05:19.215894Z 2021-12-17T14:05:19.079302Z 2021-12-17T14:05:18.877346Z 2021-12-17T14:05:18.840867Z 2021-12-17T14:05:18.275296Z 2021-12-17T14:05:18.212237Z 2021-12-17T14:05:18.201322Z 2021-12-17T14:05:18.193524Z 2021-12-17T14:05:18.065532Z 2021-12-17T14:05:18.051835Z 2021-12-17T14:05:17.599836Z 2021-12-17T14:05:17.402761Z 2021-12-17T14:05:17.378969Z 2021-12-17T14:05:16.975487Z 2021-12-17T14:05:16.926870Z 2021-12-17T14:05:16.707298Z 2021-12-17T14:05:16.417016Z 2021-12-17T14:05:16.142079Z 2021-12-17T14:05:15.956726Z 2021-12-17T14:05:15.948236Z 2021-12-17T14:05:15.457156Z 2021-12-17T14:05:15.429364Z 2021-12-17T14:05:15.378211Z 2021-12-17T14:05:15.001168Z 2021-12-17T14:05:14.807341Z 2021-12-17T14:05:14.765671Z 2021-12-17T14:05:14.546415Z 2021-12-17T14:05:14.514251Z 2021-12-17T14:05:14.170209Z 2021-12-17T14:05:14.151319Z 2021-12-17T14:05:14.086191Z 2021-12-17T14:05:13.910926Z 2021-12-17T14:05:13.842008Z 2021-12-17T14:05:13.797047Z 2021-12-17T14:05:13.678230Z 2021-12-17T14:05:13.248389Z 2021-12-17T14:05:13.065180Z 2021-12-17T14:05:12.807248Z 2021-12-17T14:05:12.544797Z 2021-12-17T14:05:12.449863Z 2021-12-17T14:05:12.328954Z 2021-12-17T14:05:12.245583Z 2021-12-17T14:05:12.215491Z 2021-12-17T14:05:12.161961Z 2021-12-17T14:05:11.800834Z 2021-12-17T14:05:11.750731Z 2021-12-17T14:05:11.502628Z 2021-12-17T14:05:11.235357Z 2021-12-17T14:05:11.206664Z 2021-12-17T14:05:11.151979Z 2021-12-17T14:05:11.136664Z 2021-12-17T14:05:11.071897Z 2021-12-17T14:05:10.993117Z 2021-12-17T14:05:10.960485Z 2021-12-17T14:05:10.756564Z 2021-12-17T14:05:10.714875Z 2021-12-17T14:05:10.588999Z 2021-12-17T14:05:10.171456Z 2021-12-17T14:05:10.140744Z 2021-12-17T14:05:10.057281Z 2021-12-17T14:05:10.055971Z 2021-12-17T14:05:10.026375Z 2021-12-17T14:05:09.937812Z 2021-12-17T14:05:09.512105Z 2021-12-17T14:05:09.412416Z 2021-12-17T14:05:08.605317Z 2021-12-17T14:10:10.455613Z 2021-12-17T14:10:10.098495Z 2021-12-17T14:10:09.737726Z 2021-12-17T14:10:09.581627Z 2021-12-17T14:10:08.966622Z 2021-12-17T14:10:08.480500Z 2021-12-17T14:10:07.710839Z 2021-12-17T14:10:07.339319Z 2021-12-17T14:10:07.120784Z 2021-12-17T14:10:07.038335Z 2021-12-17T14:10:06.967148Z 2021-12-17T14:10:06.900092Z 2021-12-17T14:10:06.883968Z 2021-12-17T14:10:06.838218Z 2021-12-17T14:10:06.616577Z 2021-12-17T14:10:06.322315Z 2021-12-17T14:10:06.009213Z 2021-12-17T14:10:05.905611Z 2021-12-17T14:10:05.831558Z 2021-12-17T14:10:05.601929Z 2021-12-17T14:10:05.587268Z 2021-12-17T14:10:05.547367Z 2021-12-17T14:10:05.436365Z 2021-12-17T14:10:05.428729Z 2021-12-17T14:10:05.359481Z 2021-12-17T14:10:05.297366Z 2021-12-17T14:10:04.855217Z 2021-12-17T14:10:04.744852Z 2021-12-17T14:10:04.071360Z 2021-12-17T14:10:03.794724Z 2021-12-17T14:10:03.504712Z 2021-12-17T14:10:03.111322Z 2021-12-17T14:10:02.932593Z 2021-12-17T14:10:02.772219Z 2021-12-17T14:10:02.406393Z 2021-12-17T14:10:02.352728Z 2021-12-17T14:10:02.010967Z 2021-12-17T14:10:01.882273Z 2021-12-17T14:10:01.642749Z 2021-12-17T14:10:01.450749Z 2021-12-17T14:10:01.302153Z 2021-12-17T14:10:01.246851Z 2021-12-17T14:10:01.241844Z 2021-12-17T14:10:01.131407Z 2021-12-17T14:10:01.067573Z 2021-12-17T14:10:01.064815Z 2021-12-17T14:10:01.051477Z 2021-12-17T14:10:00.833597Z 2021-12-17T14:10:00.799466Z 2021-12-17T14:10:00.535444Z 2021-12-17T14:10:00.360404Z 2021-12-17T14:10:00.280423Z 2021-12-17T14:10:00.083040Z 2021-12-17T14:10:00.024500Z 2021-12-17T14:09:59.852772Z 2021-12-17T14:09:59.713583Z 2021-12-17T14:09:59.601034Z 2021-12-17T14:09:59.240665Z 2021-12-17T14:09:59.188067Z 2021-12-17T14:09:58.984472Z 2021-12-17T14:09:58.797829Z 2021-12-17T14:09:58.357527Z 2021-12-17T14:09:58.241341Z 2021-12-17T14:09:58.209398Z 2021-12-17T14:09:57.971160Z 2021-12-17T14:09:57.755398Z 2021-12-17T14:09:57.625989Z 2021-12-17T14:09:57.315669Z 2021-12-17T14:09:57.269342Z 2021-12-17T14:09:57.177917Z 2021-12-17T14:09:56.942858Z 2021-12-17T14:09:56.961428Z 2021-12-17T14:09:56.748700Z 2021-12-17T14:09:56.683417Z 2021-12-17T14:09:56.615720Z 2021-12-17T14:09:56.532741Z 2021-12-17T14:09:56.326213Z 2021-12-17T14:09:55.997777Z 2021-12-17T14:09:55.994621Z 2021-12-17T14:09:55.938546Z 2021-12-17T14:09:55.824765Z 2021-12-17T14:09:55.753297Z 2021-12-17T14:09:55.555953Z 2021-12-17T14:09:55.536833Z 2021-12-17T14:09:55.512518Z 2021-12-17T14:09:55.202546Z 2021-12-17T14:09:55.198110Z 2021-12-17T14:09:55.183521Z 2021-12-17T14:09:55.103526Z 2021-12-17T14:09:55.064058Z 2021-12-17T14:09:54.672081Z 2021-12-17T14:09:54.567371Z 2021-12-17T14:09:54.442278Z 2021-12-17T14:09:54.297702Z 2021-12-17T14:09:54.155653Z 2021-12-17T14:09:53.993401Z 2021-12-17T14:09:53.545748Z 2021-12-17T14:09:53.191076Z 2021-12-17T14:09:52.854012Z 2021-12-17T14:09:52.407696Z 2021-12-17T14:14:58.092804Z 2021-12-17T14:14:57.849370Z 2021-12-17T14:14:57.807329Z 2021-12-17T14:14:57.544753Z 2021-12-17T14:14:57.128656Z 2021-12-17T14:14:56.925143Z 2021-12-17T14:14:56.499514Z 2021-12-17T14:14:56.144979Z 2021-12-17T14:14:56.024812Z 2021-12-17T14:14:56.004427Z 2021-12-17T14:14:55.885715Z 2021-12-17T14:14:55.841888Z 2021-12-17T14:14:55.795398Z 2021-12-17T14:14:55.731485Z 2021-12-17T14:14:55.632650Z 2021-12-17T14:14:55.339048Z 2021-12-17T14:14:55.309149Z 2021-12-17T14:14:55.241117Z 2021-12-17T14:14:55.181635Z 2021-12-17T14:14:55.120580Z 2021-12-17T14:14:55.112832Z 2021-12-17T14:14:55.064727Z 2021-12-17T14:14:54.983212Z 2021-12-17T14:14:54.761750Z 2021-12-17T14:14:54.675543Z 2021-12-17T14:14:54.641079Z 2021-12-17T14:14:54.541070Z 2021-12-17T14:14:54.509235Z 2021-12-17T14:14:54.197497Z 2021-12-17T14:14:53.943745Z 2021-12-17T14:14:53.885418Z 2021-12-17T14:14:53.739947Z 2021-12-17T14:14:53.617665Z 2021-12-17T14:14:53.561438Z 2021-12-17T14:14:53.459884Z 2021-12-17T14:14:53.201453Z 2021-12-17T14:14:53.108288Z 2021-12-17T14:14:53.028208Z 2021-12-17T14:14:52.844418Z 2021-12-17T14:14:52.836709Z 2021-12-17T14:14:52.676330Z 2021-12-17T14:14:52.297718Z 2021-12-17T14:14:52.277072Z 2021-12-17T14:14:52.228258Z 2021-12-17T14:14:52.051330Z 2021-12-17T14:14:51.340987Z 2021-12-17T14:14:51.176092Z 2021-12-17T14:14:51.101833Z 2021-12-17T14:14:51.082839Z 2021-12-17T14:14:51.013540Z 2021-12-17T14:14:50.989306Z 2021-12-17T14:14:50.966480Z 2021-12-17T14:14:50.936648Z 2021-12-17T14:14:50.917472Z 2021-12-17T14:14:50.783139Z 2021-12-17T14:14:50.685361Z 2021-12-17T14:14:50.618171Z 2021-12-17T14:14:50.429175Z 2021-12-17T14:14:50.370244Z 2021-12-17T14:14:50.365005Z 2021-12-17T14:14:50.038628Z 2021-12-17T14:14:49.906042Z 2021-12-17T14:14:49.868566Z 2021-12-17T14:14:49.836534Z 2021-12-17T14:14:49.604262Z 2021-12-17T14:14:49.100243Z 2021-12-17T14:14:49.063973Z 2021-12-17T14:14:49.052184Z 2021-12-17T14:14:48.909264Z 2021-12-17T14:14:48.896060Z 2021-12-17T14:14:48.741655Z 2021-12-17T14:14:48.627986Z 2021-12-17T14:14:48.429467Z 2021-12-17T14:14:48.333989Z 2021-12-17T14:14:48.053361Z 2021-12-17T14:14:48.051995Z 2021-12-17T14:14:48.005793Z 2021-12-17T14:14:47.938327Z 2021-12-17T14:14:47.922441Z 2021-12-17T14:14:47.881802Z 2021-12-17T14:14:47.861434Z 2021-12-17T14:14:47.785733Z 2021-12-17T14:14:47.651169Z 2021-12-17T14:14:47.507049Z 2021-12-17T14:14:47.456788Z 2021-12-17T14:14:47.407677Z 2021-12-17T14:14:47.378402Z 2021-12-17T14:14:47.237381Z 2021-12-17T14:14:47.225636Z 2021-12-17T14:14:47.196630Z 2021-12-17T14:14:47.174152Z 2021-12-17T14:14:46.984117Z 2021-12-17T14:14:46.978623Z 2021-12-17T14:14:46.972479Z 2021-12-17T14:14:46.648155Z 2021-12-17T14:14:46.611165Z 2021-12-17T14:14:46.409106Z 2021-12-17T14:14:46.293593Z 2021-12-17T14:14:46.203519Z 2021-12-17T14:14:45.676133Z 2021-12-17T14:19:49.501212Z 2021-12-17T14:19:49.046478Z 2021-12-17T14:19:48.426222Z 2021-12-17T14:19:48.212230Z 2021-12-17T14:19:48.062573Z 2021-12-17T14:19:47.676370Z 2021-12-17T14:19:47.645918Z 2021-12-17T14:19:47.471335Z 2021-12-17T14:19:46.942939Z 2021-12-17T14:19:46.240637Z 2021-12-17T14:19:45.830525Z 2021-12-17T14:19:45.804817Z 2021-12-17T14:19:45.796794Z 2021-12-17T14:19:45.648719Z 2021-12-17T14:19:45.568779Z 2021-12-17T14:19:45.424360Z 2021-12-17T14:19:45.386520Z 2021-12-17T14:19:45.234201Z 2021-12-17T14:19:44.907337Z 2021-12-17T14:19:44.879457Z 2021-12-17T14:19:44.731302Z 2021-12-17T14:19:44.709270Z 2021-12-17T14:19:44.659473Z 2021-12-17T14:19:44.429783Z 2021-12-17T14:19:43.872497Z 2021-12-17T14:19:43.603749Z 2021-12-17T14:19:43.539899Z 2021-12-17T14:19:43.438477Z 2021-12-17T14:19:43.428841Z 2021-12-17T14:19:43.358430Z 2021-12-17T14:19:43.348234Z 2021-12-17T14:19:43.316621Z 2021-12-17T14:19:43.057589Z 2021-12-17T14:19:42.998133Z 2021-12-17T14:19:42.896509Z 2021-12-17T14:19:42.895507Z 2021-12-17T14:19:42.861889Z 2021-12-17T14:19:42.543186Z 2021-12-17T14:19:42.114292Z 2021-12-17T14:19:42.097561Z 2021-12-17T14:19:42.094975Z 2021-12-17T14:19:42.077767Z 2021-12-17T14:19:42.075545Z 2021-12-17T14:19:42.016413Z 2021-12-17T14:19:41.928723Z 2021-12-17T14:19:41.553260Z 2021-12-17T14:19:41.474118Z 2021-12-17T14:19:41.379824Z 2021-12-17T14:19:41.295763Z 2021-12-17T14:19:41.206009Z 2021-12-17T14:19:41.179712Z 2021-12-17T14:19:40.999635Z 2021-12-17T14:19:40.858562Z 2021-12-17T14:19:40.665090Z 2021-12-17T14:19:40.646875Z 2021-12-17T14:19:40.543912Z 2021-12-17T14:19:40.040717Z 2021-12-17T14:19:39.776115Z 2021-12-17T14:19:39.716486Z 2021-12-17T14:19:39.646817Z 2021-12-17T14:19:39.422807Z 2021-12-17T14:19:39.281197Z 2021-12-17T14:19:39.248867Z 2021-12-17T14:19:39.163194Z 2021-12-17T14:19:39.010997Z 2021-12-17T14:19:39.002675Z 2021-12-17T14:19:38.933971Z 2021-12-17T14:19:38.895134Z 2021-12-17T14:19:38.735632Z 2021-12-17T14:19:38.669602Z 2021-12-17T14:19:38.661422Z 2021-12-17T14:19:38.638314Z 2021-12-17T14:19:38.603147Z 2021-12-17T14:19:38.576382Z 2021-12-17T14:19:38.543823Z 2021-12-17T14:19:38.281490Z 2021-12-17T14:19:38.278510Z 2021-12-17T14:19:38.051668Z 2021-12-17T14:19:37.974718Z 2021-12-17T14:19:37.876188Z 2021-12-17T14:19:37.764902Z 2021-12-17T14:19:37.665550Z 2021-12-17T14:19:37.381652Z 2021-12-17T14:19:37.020220Z 2021-12-17T14:19:37.004889Z 2021-12-17T14:19:36.965431Z 2021-12-17T14:19:36.952367Z 2021-12-17T14:19:36.860001Z 2021-12-17T14:19:36.797527Z 2021-12-17T14:19:36.503314Z 2021-12-17T14:19:36.397545Z 2021-12-17T14:19:36.358560Z 2021-12-17T14:19:36.265960Z 2021-12-17T14:19:36.236606Z 2021-12-17T14:19:36.220914Z 2021-12-17T14:19:36.127194Z 2021-12-17T14:19:35.803430Z 2021-12-17T14:19:35.586598Z 2021-12-17T14:19:35.519130Z 2021-12-17T14:19:34.609515Z", + "created_diff": 19.7383654, + "ended": "2021-12-17T14:22:01.401749Z", + "error_job_ids": " []", + "event_first": "2021-12-17T14:00:40.050622Z 2021-12-17T14:01:12.315508Z 2021-12-17T14:00:40.588714Z 2021-12-17T14:01:10.276341Z 2021-12-17T14:00:38.035457Z 2021-12-17T14:01:18.137661Z 2021-12-17T14:00:33.886000Z 2021-12-17T14:01:15.363274Z 2021-12-17T14:01:48.294754Z 2021-12-17T14:01:14.353437Z 2021-12-17T14:01:21.340475Z 2021-12-17T14:01:06.468642Z 2021-12-17T14:01:56.903193Z 2021-12-17T14:01:03.172749Z 2021-12-17T14:01:38.980676Z 2021-12-17T14:01:07.679545Z 2021-12-17T14:01:07.204730Z 2021-12-17T14:01:00.548413Z 2021-12-17T14:02:05.563391Z 2021-12-17T14:00:48.038631Z 2021-12-17T14:02:14.218393Z 2021-12-17T14:00:47.468404Z 2021-12-17T14:01:30.370288Z 2021-12-17T14:00:48.127192Z 2021-12-17T14:03:03.787603Z 2021-12-17T14:00:28.224619Z 2021-12-17T14:02:53.005398Z 2021-12-17T14:00:34.656688Z 2021-12-17T14:02:43.311829Z 2021-12-17T14:00:04.630386Z 2021-12-17T14:02:33.955119Z 2021-12-17T14:00:02.860863Z 2021-12-17T14:02:23.927657Z 2021-12-17T13:59:59.787250Z 2021-12-17T14:00:11.687081Z 2021-12-17T13:59:54.995470Z 2021-12-17T14:01:47.672580Z 2021-12-17T14:01:15.551322Z 2021-12-17T14:01:39.754735Z 2021-12-17T14:01:16.383406Z 2021-12-17T14:01:36.420375Z 2021-12-17T14:01:13.513826Z 2021-12-17T14:01:36.220741Z 2021-12-17T14:01:13.945720Z 2021-12-17T14:01:38.686407Z 2021-12-17T14:01:13.321358Z 2021-12-17T14:01:37.298410Z 2021-12-17T14:01:16.623175Z 2021-12-17T14:01:41.388616Z 2021-12-17T14:01:14.400394Z 2021-12-17T14:01:44.929136Z 2021-12-17T14:01:13.688439Z 2021-12-17T14:01:43.280929Z 2021-12-17T14:01:14.850205Z 2021-12-17T14:01:37.780799Z 2021-12-17T14:01:42.150747Z 2021-12-17T14:01:38.978586Z 2021-12-17T14:01:05.817653Z 2021-12-17T14:01:39.310428Z 2021-12-17T14:01:42.329268Z 2021-12-17T14:01:00.007937Z 2021-12-17T14:01:32.257956Z 2021-12-17T14:01:10.800504Z 2021-12-17T14:01:41.069532Z 2021-12-17T14:01:10.383206Z 2021-12-17T14:01:30.703552Z 2021-12-17T14:01:01.054763Z 2021-12-17T14:01:39.941281Z 2021-12-17T14:01:02.070372Z 2021-12-17T14:01:41.257489Z 2021-12-17T14:01:08.229110Z 2021-12-17T14:01:32.922214Z 2021-12-17T14:01:04.347639Z 2021-12-17T14:01:36.420375Z 2021-12-17T14:01:01.320818Z 2021-12-17T14:01:33.184425Z 2021-12-17T14:01:11.556262Z 2021-12-17T14:01:33.748725Z 2021-12-17T14:00:57.743638Z 2021-12-17T14:01:31.802226Z 2021-12-17T14:01:01.827494Z 2021-12-17T14:01:39.798229Z 2021-12-17T14:00:57.417671Z 2021-12-17T14:01:33.184425Z 2021-12-17T14:00:53.462617Z 2021-12-17T14:01:31.394819Z 2021-12-17T14:01:00.065262Z 2021-12-17T14:01:19.416370Z 2021-12-17T14:00:59.613516Z 2021-12-17T14:01:27.879576Z 2021-12-17T14:00:54.987640Z 2021-12-17T14:01:20.694435Z 2021-12-17T14:00:57.262760Z 2021-12-17T14:01:25.895994Z 2021-12-17T14:00:57.164541Z 2021-12-17T14:01:20.589369Z 2021-12-17T14:00:41.055408Z 2021-12-17T14:01:28.675631Z 2021-12-17T14:00:47.109261Z 2021-12-17T14:01:29.356255Z 2021-12-17T14:07:19.182607Z 2021-12-17T14:07:09.018822Z 2021-12-17T14:07:16.654369Z 2021-12-17T14:07:10.138484Z 2021-12-17T14:07:14.210996Z 2021-12-17T14:07:06.449230Z 2021-12-17T14:07:17.555854Z 2021-12-17T14:07:04.684336Z 2021-12-17T14:07:17.890366Z 2021-12-17T14:07:04.942031Z 2021-12-17T14:07:16.627365Z 2021-12-17T14:07:06.999343Z 2021-12-17T14:07:09.399687Z 2021-12-17T14:07:05.504648Z 2021-12-17T14:07:09.192372Z 2021-12-17T14:07:01.317695Z 2021-12-17T14:07:07.626581Z 2021-12-17T14:07:12.811115Z 2021-12-17T14:07:00.836892Z 2021-12-17T14:07:02.055691Z 2021-12-17T14:07:08.490527Z 2021-12-17T14:07:05.945683Z 2021-12-17T14:07:04.235227Z 2021-12-17T14:07:02.410238Z 2021-12-17T14:07:14.608787Z 2021-12-17T14:07:01.776267Z 2021-12-17T14:06:59.486513Z 2021-12-17T14:06:57.007524Z 2021-12-17T14:07:08.937132Z 2021-12-17T14:06:58.247899Z 2021-12-17T14:07:10.743335Z 2021-12-17T14:07:01.807493Z 2021-12-17T14:07:10.407773Z 2021-12-17T14:06:58.816486Z 2021-12-17T14:07:10.407773Z 2021-12-17T14:06:55.516011Z 2021-12-17T14:07:10.743335Z 2021-12-17T14:06:57.283312Z 2021-12-17T14:07:06.596114Z 2021-12-17T14:06:52.112518Z 2021-12-17T14:07:02.517452Z 2021-12-17T14:06:47.724636Z 2021-12-17T14:06:59.976759Z 2021-12-17T14:06:41.871321Z 2021-12-17T14:06:55.426553Z 2021-12-17T14:06:43.782109Z 2021-12-17T14:06:58.285411Z 2021-12-17T14:06:44.705144Z 2021-12-17T14:06:54.934010Z 2021-12-17T14:06:48.298957Z 2021-12-17T14:06:59.739614Z 2021-12-17T14:06:39.419117Z 2021-12-17T14:06:49.353811Z 2021-12-17T14:06:45.154578Z 2021-12-17T14:06:58.031388Z 2021-12-17T14:06:37.760323Z 2021-12-17T14:06:58.538805Z 2021-12-17T14:06:38.096561Z 2021-12-17T14:06:59.486513Z 2021-12-17T14:06:33.189021Z 2021-12-17T14:06:54.171306Z 2021-12-17T14:06:32.258537Z 2021-12-17T14:06:46.033552Z 2021-12-17T14:06:23.821698Z 2021-12-17T14:06:48.559417Z 2021-12-17T14:06:15.266559Z 2021-12-17T14:06:45.090844Z 2021-12-17T14:06:22.678980Z 2021-12-17T14:05:52.181641Z 2021-12-17T14:06:07.346561Z 2021-12-17T14:05:49.663282Z 2021-12-17T14:06:12.350212Z 2021-12-17T14:05:48.452242Z 2021-12-17T14:06:01.993756Z 2021-12-17T14:05:48.004181Z 2021-12-17T14:06:07.523790Z 2021-12-17T14:05:47.476664Z 2021-12-17T14:06:05.492731Z 2021-12-17T14:05:45.996751Z 2021-12-17T14:05:58.503680Z 2021-12-17T14:05:44.460678Z 2021-12-17T14:05:57.712158Z 2021-12-17T14:05:42.887955Z 2021-12-17T14:05:52.816117Z 2021-12-17T14:05:39.442155Z 2021-12-17T14:05:46.177106Z 2021-12-17T14:05:37.141430Z 2021-12-17T14:05:50.647423Z 2021-12-17T14:05:37.059549Z 2021-12-17T14:05:53.194777Z 2021-12-17T14:05:27.359062Z 2021-12-17T14:05:34.138689Z 2021-12-17T14:05:27.827402Z 2021-12-17T14:05:31.544511Z 2021-12-17T14:05:27.611211Z 2021-12-17T14:05:32.006677Z 2021-12-17T14:05:26.345693Z 2021-12-17T14:05:27.657031Z 2021-12-17T14:05:22.237384Z 2021-12-17T14:05:18.281105Z 2021-12-17T14:11:58.017149Z 2021-12-17T14:11:58.795314Z 2021-12-17T14:11:50.347451Z 2021-12-17T14:12:06.189846Z 2021-12-17T14:11:58.248005Z 2021-12-17T14:12:07.734972Z 2021-12-17T14:11:54.362366Z 2021-12-17T14:12:00.901608Z 2021-12-17T14:11:57.017484Z 2021-12-17T14:12:06.453641Z 2021-12-17T14:12:00.080436Z 2021-12-17T14:12:02.633383Z 2021-12-17T14:11:54.362366Z 2021-12-17T14:11:56.072457Z 2021-12-17T14:11:54.362366Z 2021-12-17T14:11:59.105979Z 2021-12-17T14:11:54.801940Z 2021-12-17T14:11:55.201423Z 2021-12-17T14:11:54.362366Z 2021-12-17T14:12:06.996527Z 2021-12-17T14:11:51.885973Z 2021-12-17T14:11:59.326484Z 2021-12-17T14:11:46.755537Z 2021-12-17T14:12:01.630722Z 2021-12-17T14:11:52.635835Z 2021-12-17T14:12:05.682778Z 2021-12-17T14:11:56.211610Z 2021-12-17T14:12:00.873458Z 2021-12-17T14:11:48.283843Z 2021-12-17T14:12:01.883509Z 2021-12-17T14:11:49.286024Z 2021-12-17T14:11:53.525699Z 2021-12-17T14:11:50.328281Z 2021-12-17T14:11:56.019011Z 2021-12-17T14:11:49.542547Z 2021-12-17T14:12:03.427392Z 2021-12-17T14:11:51.073840Z 2021-12-17T14:11:56.040285Z 2021-12-17T14:11:41.402466Z 2021-12-17T14:11:51.204924Z 2021-12-17T14:11:43.458945Z 2021-12-17T14:11:46.409382Z 2021-12-17T14:11:47.772426Z 2021-12-17T14:12:00.776570Z 2021-12-17T14:11:41.403768Z 2021-12-17T14:11:51.763702Z 2021-12-17T14:11:42.772371Z 2021-12-17T14:11:56.017800Z 2021-12-17T14:11:43.979100Z 2021-12-17T14:11:51.691207Z 2021-12-17T14:11:43.444350Z 2021-12-17T14:11:28.126542Z 2021-12-17T14:11:39.650819Z 2021-12-17T14:11:52.479383Z 2021-12-17T14:11:36.017214Z 2021-12-17T14:11:34.310525Z 2021-12-17T14:11:26.079334Z 2021-12-17T14:11:41.436128Z 2021-12-17T14:11:20.422446Z 2021-12-17T14:11:29.307372Z 2021-12-17T14:11:18.804399Z 2021-12-17T14:11:39.915747Z 2021-12-17T14:11:26.759961Z 2021-12-17T14:11:35.360354Z 2021-12-17T14:11:25.651733Z 2021-12-17T14:11:34.310525Z 2021-12-17T14:11:19.313662Z 2021-12-17T14:11:34.063679Z 2021-12-17T14:11:25.074452Z 2021-12-17T14:11:13.590635Z 2021-12-17T14:11:25.067539Z 2021-12-17T14:10:59.152631Z 2021-12-17T14:11:24.140199Z 2021-12-17T14:11:11.852437Z 2021-12-17T14:11:15.123607Z 2021-12-17T14:11:11.494561Z 2021-12-17T14:11:08.240471Z 2021-12-17T14:11:02.746058Z 2021-12-17T14:11:14.077131Z 2021-12-17T14:11:20.843596Z 2021-12-17T14:11:11.366458Z 2021-12-17T14:11:12.621113Z 2021-12-17T14:10:56.009930Z 2021-12-17T14:11:08.381295Z 2021-12-17T14:10:45.728648Z 2021-12-17T14:10:55.879067Z 2021-12-17T14:10:46.750734Z 2021-12-17T14:10:55.702745Z 2021-12-17T14:10:45.447720Z 2021-12-17T14:10:43.752868Z 2021-12-17T14:10:17.117723Z 2021-12-17T14:10:17.398508Z 2021-12-17T14:10:15.052123Z 2021-12-17T14:10:15.676411Z 2021-12-17T14:10:13.917411Z 2021-12-17T14:10:15.199747Z 2021-12-17T14:10:07.930580Z 2021-12-17T14:10:07.503583Z 2021-12-17T14:10:03.539722Z 2021-12-17T14:10:03.748779Z 2021-12-17T14:16:55.108347Z 2021-12-17T14:16:57.001626Z 2021-12-17T14:16:47.488314Z 2021-12-17T14:16:59.241667Z 2021-12-17T14:16:52.196512Z 2021-12-17T14:16:57.386552Z 2021-12-17T14:16:48.252475Z 2021-12-17T14:16:50.977277Z 2021-12-17T14:16:51.861539Z 2021-12-17T14:16:59.010039Z 2021-12-17T14:16:51.921428Z 2021-12-17T14:16:59.090374Z 2021-12-17T14:16:56.429416Z 2021-12-17T14:16:58.270264Z 2021-12-17T14:16:48.252475Z 2021-12-17T14:16:54.826541Z 2021-12-17T14:16:48.496411Z 2021-12-17T14:16:59.010039Z 2021-12-17T14:16:47.976333Z 2021-12-17T14:16:49.755877Z 2021-12-17T14:16:52.700955Z 2021-12-17T14:16:58.723259Z 2021-12-17T14:16:52.980331Z 2021-12-17T14:16:55.823613Z 2021-12-17T14:16:47.001545Z 2021-12-17T14:16:54.109524Z 2021-12-17T14:16:53.314368Z 2021-12-17T14:16:49.293379Z 2021-12-17T14:16:47.681712Z 2021-12-17T14:16:44.266208Z 2021-12-17T14:16:45.216708Z 2021-12-17T14:16:46.273094Z 2021-12-17T14:16:52.430475Z 2021-12-17T14:16:46.745715Z 2021-12-17T14:16:44.928506Z 2021-12-17T14:16:45.527493Z 2021-12-17T14:16:33.830006Z 2021-12-17T14:16:43.838344Z 2021-12-17T14:16:42.408550Z 2021-12-17T14:16:54.803595Z 2021-12-17T14:16:38.829242Z 2021-12-17T14:16:51.803036Z 2021-12-17T14:16:37.605950Z 2021-12-17T14:16:37.439704Z 2021-12-17T14:16:39.924113Z 2021-12-17T14:16:42.163725Z 2021-12-17T14:16:42.266444Z 2021-12-17T14:16:42.460576Z 2021-12-17T14:16:43.645582Z 2021-12-17T14:16:55.342986Z 2021-12-17T14:16:38.114386Z 2021-12-17T14:16:44.157435Z 2021-12-17T14:16:38.422502Z 2021-12-17T14:16:37.699004Z 2021-12-17T14:16:38.114386Z 2021-12-17T14:16:37.699004Z 2021-12-17T14:16:41.637301Z 2021-12-17T14:16:42.460576Z 2021-12-17T14:16:34.009154Z 2021-12-17T14:16:34.316836Z 2021-12-17T14:16:26.894591Z 2021-12-17T14:16:33.309680Z 2021-12-17T14:16:28.203469Z 2021-12-17T14:16:36.676763Z 2021-12-17T14:16:26.323049Z 2021-12-17T14:16:00.577592Z 2021-12-17T14:15:57.547812Z 2021-12-17T14:16:19.317715Z 2021-12-17T14:16:11.383006Z 2021-12-17T14:16:14.710164Z 2021-12-17T14:16:15.501807Z 2021-12-17T14:16:18.369218Z 2021-12-17T14:15:48.435774Z 2021-12-17T14:16:06.369061Z 2021-12-17T14:16:02.180804Z 2021-12-17T14:16:11.121335Z 2021-12-17T14:16:07.527923Z 2021-12-17T14:16:03.369382Z 2021-12-17T14:16:02.653595Z 2021-12-17T14:16:01.849529Z 2021-12-17T14:15:55.518843Z 2021-12-17T14:16:04.782334Z 2021-12-17T14:15:48.949048Z 2021-12-17T14:15:50.568282Z 2021-12-17T14:15:51.339161Z 2021-12-17T14:15:51.555929Z 2021-12-17T14:15:49.952620Z 2021-12-17T14:15:34.401045Z 2021-12-17T14:15:43.454189Z 2021-12-17T14:15:33.989975Z 2021-12-17T14:15:49.406770Z 2021-12-17T14:15:25.239802Z 2021-12-17T14:15:28.478066Z 2021-12-17T14:15:21.227870Z 2021-12-17T14:15:26.701643Z 2021-12-17T14:15:26.068025Z 2021-12-17T14:15:08.352483Z 2021-12-17T14:15:09.932065Z 2021-12-17T14:15:00.338025Z 2021-12-17T14:14:55.898690Z 2021-12-17T14:21:55.885178Z 2021-12-17T14:21:43.534919Z 2021-12-17T14:21:58.917119Z 2021-12-17T14:21:41.650858Z 2021-12-17T14:21:57.949925Z 2021-12-17T14:21:40.199540Z 2021-12-17T14:21:59.177346Z 2021-12-17T14:21:40.438375Z 2021-12-17T14:21:59.592704Z 2021-12-17T14:21:39.446526Z 2021-12-17T14:21:54.831412Z 2021-12-17T14:21:43.638737Z 2021-12-17T14:21:41.768259Z 2021-12-17T14:21:28.860480Z 2021-12-17T14:21:45.249686Z 2021-12-17T14:21:27.102116Z 2021-12-17T14:21:47.620376Z 2021-12-17T14:21:31.018742Z 2021-12-17T14:21:48.189431Z 2021-12-17T14:21:53.411475Z 2021-12-17T14:21:31.426391Z 2021-12-17T14:21:48.091550Z 2021-12-17T14:21:26.768225Z 2021-12-17T14:21:50.672438Z 2021-12-17T14:21:36.139546Z 2021-12-17T14:21:50.924481Z 2021-12-17T14:21:31.639117Z 2021-12-17T14:21:51.223073Z 2021-12-17T14:21:30.654072Z 2021-12-17T14:21:45.665005Z 2021-12-17T14:21:32.449764Z 2021-12-17T14:21:32.134067Z 2021-12-17T14:21:39.929317Z 2021-12-17T14:21:28.476328Z 2021-12-17T14:21:45.110002Z 2021-12-17T14:21:26.323653Z 2021-12-17T14:21:38.321997Z 2021-12-17T14:21:23.034463Z 2021-12-17T14:21:49.894518Z 2021-12-17T14:21:58.711771Z 2021-12-17T14:21:29.892494Z 2021-12-17T14:21:45.769344Z 2021-12-17T14:21:27.774062Z 2021-12-17T14:21:41.792295Z 2021-12-17T14:21:32.718528Z 2021-12-17T14:21:42.177686Z 2021-12-17T14:21:25.236587Z 2021-12-17T14:21:43.031483Z 2021-12-17T14:21:28.026467Z 2021-12-17T14:21:38.835457Z 2021-12-17T14:21:24.591635Z 2021-12-17T14:21:39.589822Z 2021-12-17T14:21:04.451608Z 2021-12-17T14:21:28.220707Z 2021-12-17T14:21:18.453743Z 2021-12-17T14:21:31.390697Z 2021-12-17T14:21:16.524137Z 2021-12-17T14:21:27.417375Z 2021-12-17T14:21:16.629614Z 2021-12-17T14:21:25.970690Z 2021-12-17T14:21:15.341544Z 2021-12-17T14:21:25.298376Z 2021-12-17T14:21:10.242603Z 2021-12-17T14:21:28.221626Z 2021-12-17T14:21:04.549953Z 2021-12-17T14:21:27.496499Z 2021-12-17T14:21:05.585019Z 2021-12-17T14:21:23.016502Z 2021-12-17T14:20:58.486796Z 2021-12-17T14:21:20.439821Z 2021-12-17T14:21:11.365556Z 2021-12-17T14:21:12.986341Z 2021-12-17T14:21:01.180615Z 2021-12-17T14:21:15.869222Z 2021-12-17T14:21:08.681527Z 2021-12-17T14:21:10.921667Z 2021-12-17T14:20:52.785796Z 2021-12-17T14:21:05.754986Z 2021-12-17T14:20:48.313848Z 2021-12-17T14:20:59.045859Z 2021-12-17T14:20:58.986048Z 2021-12-17T14:20:55.047206Z 2021-12-17T14:20:46.372304Z 2021-12-17T14:21:04.673766Z 2021-12-17T14:20:38.580857Z 2021-12-17T14:20:54.808176Z 2021-12-17T14:20:34.971849Z 2021-12-17T14:20:45.125985Z 2021-12-17T14:20:33.713667Z 2021-12-17T14:20:00.660170Z 2021-12-17T14:19:56.336797Z 2021-12-17T14:20:08.450392Z 2021-12-17T14:19:56.908456Z 2021-12-17T14:20:03.420662Z 2021-12-17T14:19:53.934338Z 2021-12-17T14:20:01.906535Z 2021-12-17T14:19:56.503185Z 2021-12-17T14:19:51.503509Z 2021-12-17T14:19:46.762919Z 2021-12-17T14:19:45.613357Z", + "event_last": "2021-12-17T14:00:44.508372Z 2021-12-17T14:01:17.260992Z 2021-12-17T14:00:44.505442Z 2021-12-17T14:01:15.154305Z 2021-12-17T14:00:42.331158Z 2021-12-17T14:01:22.700921Z 2021-12-17T14:00:37.220227Z 2021-12-17T14:01:20.645575Z 2021-12-17T14:01:48.963618Z 2021-12-17T14:01:20.188164Z 2021-12-17T14:01:21.891591Z 2021-12-17T14:01:11.379436Z 2021-12-17T14:01:58.970674Z 2021-12-17T14:01:08.257051Z 2021-12-17T14:01:41.809306Z 2021-12-17T14:01:14.671193Z 2021-12-17T14:01:10.175861Z 2021-12-17T14:01:06.373180Z 2021-12-17T14:02:05.873849Z 2021-12-17T14:00:53.322616Z 2021-12-17T14:02:14.647952Z 2021-12-17T14:00:54.095340Z 2021-12-17T14:01:30.865908Z 2021-12-17T14:00:52.782332Z 2021-12-17T14:03:06.461415Z 2021-12-17T14:00:35.620253Z 2021-12-17T14:02:53.410551Z 2021-12-17T14:00:40.615192Z 2021-12-17T14:02:44.815836Z 2021-12-17T14:00:07.225306Z 2021-12-17T14:02:34.132357Z 2021-12-17T14:00:04.900443Z 2021-12-17T14:02:24.310612Z 2021-12-17T14:00:00.974879Z 2021-12-17T14:00:13.324150Z 2021-12-17T13:59:58.417000Z 2021-12-17T14:01:48.965715Z 2021-12-17T14:01:17.641385Z 2021-12-17T14:01:45.566853Z 2021-12-17T14:01:18.247895Z 2021-12-17T14:01:40.998713Z 2021-12-17T14:01:16.131265Z 2021-12-17T14:01:41.747238Z 2021-12-17T14:01:16.402252Z 2021-12-17T14:01:43.938241Z 2021-12-17T14:01:15.973592Z 2021-12-17T14:01:41.990578Z 2021-12-17T14:01:18.247895Z 2021-12-17T14:01:47.410490Z 2021-12-17T14:01:16.506291Z 2021-12-17T14:01:47.674158Z 2021-12-17T14:01:15.954155Z 2021-12-17T14:01:47.766804Z 2021-12-17T14:01:16.948489Z 2021-12-17T14:01:41.517732Z 2021-12-17T14:01:46.881286Z 2021-12-17T14:01:43.833214Z 2021-12-17T14:01:08.690346Z 2021-12-17T14:01:44.313758Z 2021-12-17T14:01:46.594295Z 2021-12-17T14:01:03.786601Z 2021-12-17T14:01:37.580152Z 2021-12-17T14:01:13.499929Z 2021-12-17T14:01:46.183256Z 2021-12-17T14:01:13.185675Z 2021-12-17T14:01:36.367498Z 2021-12-17T14:01:04.098013Z 2021-12-17T14:01:44.819353Z 2021-12-17T14:01:06.135203Z 2021-12-17T14:01:45.799087Z 2021-12-17T14:01:12.106169Z 2021-12-17T14:01:38.337036Z 2021-12-17T14:01:07.708074Z 2021-12-17T14:01:41.878103Z 2021-12-17T14:01:05.281166Z 2021-12-17T14:01:38.650929Z 2021-12-17T14:01:14.198411Z 2021-12-17T14:01:39.281693Z 2021-12-17T14:01:01.710223Z 2021-12-17T14:01:36.520186Z 2021-12-17T14:01:06.225401Z 2021-12-17T14:01:46.328361Z 2021-12-17T14:01:01.395261Z 2021-12-17T14:01:38.645859Z 2021-12-17T14:00:56.962260Z 2021-12-17T14:01:36.529662Z 2021-12-17T14:01:05.445918Z 2021-12-17T14:01:23.551880Z 2021-12-17T14:01:03.604120Z 2021-12-17T14:01:33.440848Z 2021-12-17T14:00:58.042611Z 2021-12-17T14:01:24.898154Z 2021-12-17T14:01:00.295242Z 2021-12-17T14:01:29.658835Z 2021-12-17T14:01:00.496098Z 2021-12-17T14:01:24.898154Z 2021-12-17T14:00:45.292562Z 2021-12-17T14:01:33.729276Z 2021-12-17T14:00:50.998323Z 2021-12-17T14:01:35.453145Z 2021-12-17T14:07:21.726361Z 2021-12-17T14:07:10.024273Z 2021-12-17T14:07:19.388332Z 2021-12-17T14:07:10.628192Z 2021-12-17T14:07:17.035247Z 2021-12-17T14:07:08.920703Z 2021-12-17T14:07:19.533998Z 2021-12-17T14:07:07.675790Z 2021-12-17T14:07:19.762684Z 2021-12-17T14:07:07.839181Z 2021-12-17T14:07:18.692926Z 2021-12-17T14:07:09.318708Z 2021-12-17T14:07:14.764581Z 2021-12-17T14:07:07.936826Z 2021-12-17T14:07:12.823709Z 2021-12-17T14:07:04.706032Z 2021-12-17T14:07:11.695181Z 2021-12-17T14:07:16.535971Z 2021-12-17T14:07:03.943234Z 2021-12-17T14:07:05.309042Z 2021-12-17T14:07:13.423014Z 2021-12-17T14:07:08.840479Z 2021-12-17T14:07:09.593980Z 2021-12-17T14:07:05.858202Z 2021-12-17T14:07:17.915489Z 2021-12-17T14:07:05.042217Z 2021-12-17T14:07:05.613738Z 2021-12-17T14:07:00.956475Z 2021-12-17T14:07:12.927194Z 2021-12-17T14:07:02.319439Z 2021-12-17T14:07:13.388227Z 2021-12-17T14:07:05.252165Z 2021-12-17T14:07:14.143239Z 2021-12-17T14:07:02.391246Z 2021-12-17T14:07:15.665273Z 2021-12-17T14:06:58.132619Z 2021-12-17T14:07:15.797520Z 2021-12-17T14:07:00.695975Z 2021-12-17T14:07:11.440841Z 2021-12-17T14:06:55.235639Z 2021-12-17T14:07:06.775294Z 2021-12-17T14:06:51.866429Z 2021-12-17T14:07:04.985424Z 2021-12-17T14:06:45.395450Z 2021-12-17T14:07:00.222227Z 2021-12-17T14:06:47.992128Z 2021-12-17T14:07:03.404264Z 2021-12-17T14:06:48.554124Z 2021-12-17T14:06:59.832251Z 2021-12-17T14:06:53.263276Z 2021-12-17T14:07:06.497320Z 2021-12-17T14:06:43.587225Z 2021-12-17T14:06:53.845229Z 2021-12-17T14:06:49.721204Z 2021-12-17T14:07:02.675854Z 2021-12-17T14:06:41.940952Z 2021-12-17T14:07:04.663297Z 2021-12-17T14:06:42.765244Z 2021-12-17T14:07:04.235227Z 2021-12-17T14:06:37.957524Z 2021-12-17T14:06:58.698826Z 2021-12-17T14:06:36.661231Z 2021-12-17T14:06:51.715189Z 2021-12-17T14:06:28.450176Z 2021-12-17T14:06:52.354980Z 2021-12-17T14:06:20.951512Z 2021-12-17T14:06:48.994272Z 2021-12-17T14:06:28.023136Z 2021-12-17T14:05:53.628664Z 2021-12-17T14:06:14.975417Z 2021-12-17T14:05:51.451829Z 2021-12-17T14:06:18.768559Z 2021-12-17T14:05:50.323309Z 2021-12-17T14:06:07.033527Z 2021-12-17T14:05:50.008462Z 2021-12-17T14:06:14.350243Z 2021-12-17T14:05:48.938593Z 2021-12-17T14:06:12.219258Z 2021-12-17T14:05:47.249176Z 2021-12-17T14:06:03.754636Z 2021-12-17T14:05:45.764098Z 2021-12-17T14:06:02.684357Z 2021-12-17T14:05:44.140988Z 2021-12-17T14:05:57.034730Z 2021-12-17T14:05:40.848123Z 2021-12-17T14:05:48.984338Z 2021-12-17T14:05:38.783257Z 2021-12-17T14:05:53.751858Z 2021-12-17T14:05:39.123261Z 2021-12-17T14:05:56.928175Z 2021-12-17T14:05:28.145906Z 2021-12-17T14:05:36.084607Z 2021-12-17T14:05:28.661978Z 2021-12-17T14:05:33.694125Z 2021-12-17T14:05:30.150442Z 2021-12-17T14:05:34.210586Z 2021-12-17T14:05:27.089069Z 2021-12-17T14:05:30.923654Z 2021-12-17T14:05:23.834138Z 2021-12-17T14:05:20.626139Z 2021-12-17T14:12:00.011161Z 2021-12-17T14:12:05.151800Z 2021-12-17T14:11:54.754281Z 2021-12-17T14:12:09.260746Z 2021-12-17T14:12:00.328144Z 2021-12-17T14:12:10.006977Z 2021-12-17T14:11:58.437832Z 2021-12-17T14:12:05.740123Z 2021-12-17T14:11:59.173142Z 2021-12-17T14:12:09.818304Z 2021-12-17T14:12:01.632254Z 2021-12-17T14:12:06.570124Z 2021-12-17T14:11:57.441135Z 2021-12-17T14:12:01.809226Z 2021-12-17T14:11:57.232330Z 2021-12-17T14:12:03.723547Z 2021-12-17T14:11:58.644418Z 2021-12-17T14:11:59.871979Z 2021-12-17T14:11:56.936281Z 2021-12-17T14:12:09.673924Z 2021-12-17T14:11:56.940014Z 2021-12-17T14:12:04.838926Z 2021-12-17T14:11:50.904224Z 2021-12-17T14:12:05.994740Z 2021-12-17T14:11:56.678242Z 2021-12-17T14:12:09.266301Z 2021-12-17T14:11:59.397255Z 2021-12-17T14:12:05.493669Z 2021-12-17T14:11:52.005341Z 2021-12-17T14:12:06.267049Z 2021-12-17T14:11:53.662153Z 2021-12-17T14:11:59.326484Z 2021-12-17T14:11:54.516172Z 2021-12-17T14:12:00.529350Z 2021-12-17T14:11:52.674216Z 2021-12-17T14:12:06.993293Z 2021-12-17T14:11:55.158166Z 2021-12-17T14:12:01.434490Z 2021-12-17T14:11:45.570395Z 2021-12-17T14:11:55.774508Z 2021-12-17T14:11:47.026037Z 2021-12-17T14:11:51.562522Z 2021-12-17T14:11:51.446123Z 2021-12-17T14:12:05.852388Z 2021-12-17T14:11:44.992159Z 2021-12-17T14:11:56.040285Z 2021-12-17T14:11:46.610547Z 2021-12-17T14:12:00.862711Z 2021-12-17T14:11:49.000225Z 2021-12-17T14:11:56.858228Z 2021-12-17T14:11:47.289189Z 2021-12-17T14:11:33.476115Z 2021-12-17T14:11:44.773243Z 2021-12-17T14:11:57.932362Z 2021-12-17T14:11:40.286148Z 2021-12-17T14:11:41.095287Z 2021-12-17T14:11:31.362171Z 2021-12-17T14:11:46.601703Z 2021-12-17T14:11:26.080425Z 2021-12-17T14:11:35.516744Z 2021-12-17T14:11:24.424588Z 2021-12-17T14:11:46.723554Z 2021-12-17T14:11:31.632591Z 2021-12-17T14:11:41.292204Z 2021-12-17T14:11:31.125138Z 2021-12-17T14:11:40.055430Z 2021-12-17T14:11:25.651733Z 2021-12-17T14:11:38.539211Z 2021-12-17T14:11:30.366155Z 2021-12-17T14:11:18.196708Z 2021-12-17T14:11:30.200194Z 2021-12-17T14:11:06.159922Z 2021-12-17T14:11:29.383367Z 2021-12-17T14:11:18.940209Z 2021-12-17T14:11:21.064139Z 2021-12-17T14:11:17.364320Z 2021-12-17T14:11:14.182126Z 2021-12-17T14:11:10.089766Z 2021-12-17T14:11:19.578177Z 2021-12-17T14:11:27.339294Z 2021-12-17T14:11:17.217133Z 2021-12-17T14:11:20.473214Z 2021-12-17T14:11:03.444120Z 2021-12-17T14:11:15.052243Z 2021-12-17T14:10:55.953465Z 2021-12-17T14:11:03.249819Z 2021-12-17T14:10:56.431647Z 2021-12-17T14:11:02.425229Z 2021-12-17T14:10:51.576591Z 2021-12-17T14:10:52.183309Z 2021-12-17T14:10:19.635670Z 2021-12-17T14:10:20.022370Z 2021-12-17T14:10:16.565773Z 2021-12-17T14:10:18.023217Z 2021-12-17T14:10:15.200344Z 2021-12-17T14:10:16.656252Z 2021-12-17T14:10:09.217235Z 2021-12-17T14:10:09.050209Z 2021-12-17T14:10:05.074206Z 2021-12-17T14:10:05.681276Z 2021-12-17T14:16:58.920969Z 2021-12-17T14:17:01.510278Z 2021-12-17T14:16:52.884463Z 2021-12-17T14:17:02.313492Z 2021-12-17T14:16:57.022499Z 2021-12-17T14:17:01.814380Z 2021-12-17T14:16:53.058628Z 2021-12-17T14:16:55.014618Z 2021-12-17T14:16:56.232169Z 2021-12-17T14:17:02.372050Z 2021-12-17T14:16:56.518329Z 2021-12-17T14:17:02.307653Z 2021-12-17T14:16:59.097245Z 2021-12-17T14:17:02.815797Z 2021-12-17T14:16:54.060105Z 2021-12-17T14:16:59.348401Z 2021-12-17T14:16:55.035277Z 2021-12-17T14:17:03.249312Z 2021-12-17T14:16:53.090661Z 2021-12-17T14:16:55.327961Z 2021-12-17T14:16:56.751490Z 2021-12-17T14:17:02.594331Z 2021-12-17T14:16:56.722064Z 2021-12-17T14:17:01.080279Z 2021-12-17T14:16:51.053679Z 2021-12-17T14:16:58.723259Z 2021-12-17T14:16:57.094760Z 2021-12-17T14:16:53.815292Z 2021-12-17T14:16:53.986638Z 2021-12-17T14:16:49.266445Z 2021-12-17T14:16:50.725378Z 2021-12-17T14:16:51.796749Z 2021-12-17T14:16:57.719366Z 2021-12-17T14:16:51.482355Z 2021-12-17T14:16:49.996697Z 2021-12-17T14:16:51.248570Z 2021-12-17T14:16:38.608164Z 2021-12-17T14:16:49.524521Z 2021-12-17T14:16:47.778425Z 2021-12-17T14:16:58.932134Z 2021-12-17T14:16:43.359160Z 2021-12-17T14:16:55.628628Z 2021-12-17T14:16:41.910265Z 2021-12-17T14:16:42.425341Z 2021-12-17T14:16:45.249333Z 2021-12-17T14:16:48.851155Z 2021-12-17T14:16:47.988719Z 2021-12-17T14:16:47.985405Z 2021-12-17T14:16:49.597412Z 2021-12-17T14:16:59.241667Z 2021-12-17T14:16:43.217960Z 2021-12-17T14:16:49.489819Z 2021-12-17T14:16:44.440160Z 2021-12-17T14:16:43.708236Z 2021-12-17T14:16:42.476048Z 2021-12-17T14:16:42.852157Z 2021-12-17T14:16:46.464459Z 2021-12-17T14:16:47.975427Z 2021-12-17T14:16:40.238127Z 2021-12-17T14:16:39.619231Z 2021-12-17T14:16:30.747772Z 2021-12-17T14:16:37.984181Z 2021-12-17T14:16:32.300121Z 2021-12-17T14:16:41.525243Z 2021-12-17T14:16:31.010154Z 2021-12-17T14:16:08.449260Z 2021-12-17T14:16:04.232130Z 2021-12-17T14:16:24.593737Z 2021-12-17T14:16:16.238059Z 2021-12-17T14:16:21.274218Z 2021-12-17T14:16:20.384451Z 2021-12-17T14:16:24.685695Z 2021-12-17T14:15:57.187188Z 2021-12-17T14:16:12.630261Z 2021-12-17T14:16:08.605157Z 2021-12-17T14:16:17.299596Z 2021-12-17T14:16:13.469109Z 2021-12-17T14:16:12.430838Z 2021-12-17T14:16:10.170156Z 2021-12-17T14:16:09.729553Z 2021-12-17T14:16:03.126309Z 2021-12-17T14:16:10.641191Z 2021-12-17T14:15:57.323964Z 2021-12-17T14:15:58.191257Z 2021-12-17T14:16:00.817127Z 2021-12-17T14:15:59.665195Z 2021-12-17T14:15:58.851857Z 2021-12-17T14:15:44.236682Z 2021-12-17T14:15:51.273114Z 2021-12-17T14:15:43.272368Z 2021-12-17T14:15:58.790181Z 2021-12-17T14:15:31.067278Z 2021-12-17T14:15:35.071201Z 2021-12-17T14:15:26.490183Z 2021-12-17T14:15:33.484650Z 2021-12-17T14:15:32.592500Z 2021-12-17T14:15:10.861690Z 2021-12-17T14:15:11.990190Z 2021-12-17T14:15:01.602374Z 2021-12-17T14:14:57.211671Z 2021-12-17T14:21:58.725573Z 2021-12-17T14:21:44.308933Z 2021-12-17T14:22:01.058127Z 2021-12-17T14:21:43.534919Z 2021-12-17T14:22:00.054104Z 2021-12-17T14:21:42.283632Z 2021-12-17T14:22:00.881894Z 2021-12-17T14:21:42.454313Z 2021-12-17T14:22:01.401749Z 2021-12-17T14:21:41.528503Z 2021-12-17T14:21:58.232265Z 2021-12-17T14:21:49.690193Z 2021-12-17T14:21:43.710014Z 2021-12-17T14:21:34.407526Z 2021-12-17T14:21:50.707202Z 2021-12-17T14:21:29.645211Z 2021-12-17T14:21:52.511037Z 2021-12-17T14:21:35.069238Z 2021-12-17T14:21:54.672274Z 2021-12-17T14:21:56.964526Z 2021-12-17T14:21:35.438675Z 2021-12-17T14:21:54.659910Z 2021-12-17T14:21:32.205824Z 2021-12-17T14:21:55.079470Z 2021-12-17T14:21:38.351276Z 2021-12-17T14:21:54.648578Z 2021-12-17T14:21:35.850652Z 2021-12-17T14:21:55.104342Z 2021-12-17T14:21:34.654153Z 2021-12-17T14:21:50.073332Z 2021-12-17T14:21:36.958543Z 2021-12-17T14:21:38.779979Z 2021-12-17T14:21:41.528503Z 2021-12-17T14:21:32.803071Z 2021-12-17T14:21:50.552682Z 2021-12-17T14:21:30.044216Z 2021-12-17T14:21:44.309526Z 2021-12-17T14:21:26.298414Z 2021-12-17T14:21:55.104342Z 2021-12-17T14:22:00.657384Z 2021-12-17T14:21:34.813131Z 2021-12-17T14:21:50.681205Z 2021-12-17T14:21:32.351889Z 2021-12-17T14:21:45.865226Z 2021-12-17T14:21:37.643241Z 2021-12-17T14:21:48.279224Z 2021-12-17T14:21:30.661500Z 2021-12-17T14:21:48.692296Z 2021-12-17T14:21:32.396150Z 2021-12-17T14:21:44.675834Z 2021-12-17T14:21:29.317141Z 2021-12-17T14:21:45.216261Z 2021-12-17T14:21:09.995258Z 2021-12-17T14:21:33.907201Z 2021-12-17T14:21:23.473498Z 2021-12-17T14:21:37.123140Z 2021-12-17T14:21:21.442381Z 2021-12-17T14:21:33.476738Z 2021-12-17T14:21:22.457445Z 2021-12-17T14:21:31.525142Z 2021-12-17T14:21:20.697230Z 2021-12-17T14:21:31.263239Z 2021-12-17T14:21:16.728541Z 2021-12-17T14:21:33.907201Z 2021-12-17T14:21:10.076957Z 2021-12-17T14:21:34.083593Z 2021-12-17T14:21:10.718541Z 2021-12-17T14:21:28.431274Z 2021-12-17T14:21:06.348365Z 2021-12-17T14:21:27.218296Z 2021-12-17T14:21:15.757146Z 2021-12-17T14:21:19.553837Z 2021-12-17T14:21:07.015110Z 2021-12-17T14:21:21.363619Z 2021-12-17T14:21:14.380849Z 2021-12-17T14:21:16.507235Z 2021-12-17T14:20:57.535668Z 2021-12-17T14:21:14.033244Z 2021-12-17T14:20:54.305223Z 2021-12-17T14:21:05.091231Z 2021-12-17T14:21:04.328421Z 2021-12-17T14:21:00.507697Z 2021-12-17T14:20:53.150094Z 2021-12-17T14:21:12.280707Z 2021-12-17T14:20:43.109300Z 2021-12-17T14:21:02.285311Z 2021-12-17T14:20:41.148134Z 2021-12-17T14:20:54.064224Z 2021-12-17T14:20:39.286853Z 2021-12-17T14:20:03.961059Z 2021-12-17T14:19:58.604751Z 2021-12-17T14:20:12.924276Z 2021-12-17T14:19:59.020207Z 2021-12-17T14:20:06.656279Z 2021-12-17T14:19:55.235471Z 2021-12-17T14:20:05.305841Z 2021-12-17T14:19:58.764426Z 2021-12-17T14:19:52.483546Z 2021-12-17T14:19:47.602592Z 2021-12-17T14:19:48.862560Z", + "failed_job_ids": " []", + "finished": "2021-12-17T14:00:50.564785Z 2021-12-17T14:01:24.152671Z 2021-12-17T14:00:50.544027Z 2021-12-17T14:01:22.467412Z 2021-12-17T14:00:46.173088Z 2021-12-17T14:01:28.114629Z 2021-12-17T14:00:43.386123Z 2021-12-17T14:01:28.112239Z 2021-12-17T14:01:52.399648Z 2021-12-17T14:01:25.261148Z 2021-12-17T14:01:26.174797Z 2021-12-17T14:01:15.781898Z 2021-12-17T14:02:00.993273Z 2021-12-17T14:01:14.826247Z 2021-12-17T14:01:43.911834Z 2021-12-17T14:01:21.522968Z 2021-12-17T14:01:14.902962Z 2021-12-17T14:01:11.637908Z 2021-12-17T14:02:09.561307Z 2021-12-17T14:00:59.479125Z 2021-12-17T14:02:18.073816Z 2021-12-17T14:01:00.332168Z 2021-12-17T14:01:34.256692Z 2021-12-17T14:00:58.513985Z 2021-12-17T14:03:08.119241Z 2021-12-17T14:00:42.688074Z 2021-12-17T14:02:57.091700Z 2021-12-17T14:00:46.416158Z 2021-12-17T14:02:47.464135Z 2021-12-17T14:00:10.671507Z 2021-12-17T14:02:37.818777Z 2021-12-17T14:00:09.198987Z 2021-12-17T14:02:27.766299Z 2021-12-17T14:00:04.946618Z 2021-12-17T14:00:17.503689Z 2021-12-17T14:00:00.946817Z 2021-12-17T14:01:52.749513Z 2021-12-17T14:01:21.810662Z 2021-12-17T14:01:50.552197Z 2021-12-17T14:01:22.459532Z 2021-12-17T14:01:46.060392Z 2021-12-17T14:01:21.300624Z 2021-12-17T14:01:48.100071Z 2021-12-17T14:01:21.398557Z 2021-12-17T14:01:50.063334Z 2021-12-17T14:01:21.039580Z 2021-12-17T14:01:48.461403Z 2021-12-17T14:01:21.813922Z 2021-12-17T14:01:52.319973Z 2021-12-17T14:01:21.711001Z 2021-12-17T14:01:52.169908Z 2021-12-17T14:01:20.435273Z 2021-12-17T14:01:51.517552Z 2021-12-17T14:01:21.085158Z 2021-12-17T14:01:49.483495Z 2021-12-17T14:01:51.458790Z 2021-12-17T14:01:49.765061Z 2021-12-17T14:01:12.953800Z 2021-12-17T14:01:50.022402Z 2021-12-17T14:01:51.157223Z 2021-12-17T14:01:09.172111Z 2021-12-17T14:01:44.532384Z 2021-12-17T14:01:18.430210Z 2021-12-17T14:01:52.057820Z 2021-12-17T14:01:17.824513Z 2021-12-17T14:01:43.132787Z 2021-12-17T14:01:08.366306Z 2021-12-17T14:01:50.641593Z 2021-12-17T14:01:11.890264Z 2021-12-17T14:01:52.168109Z 2021-12-17T14:01:15.794182Z 2021-12-17T14:01:45.668535Z 2021-12-17T14:01:13.182474Z 2021-12-17T14:01:47.666087Z 2021-12-17T14:01:11.010125Z 2021-12-17T14:01:43.511400Z 2021-12-17T14:01:18.876145Z 2021-12-17T14:01:43.998291Z 2021-12-17T14:01:05.480793Z 2021-12-17T14:01:42.237107Z 2021-12-17T14:01:11.991490Z 2021-12-17T14:01:50.459751Z 2021-12-17T14:01:06.062714Z 2021-12-17T14:01:45.513812Z 2021-12-17T14:01:02.345386Z 2021-12-17T14:01:42.137066Z 2021-12-17T14:01:09.956680Z 2021-12-17T14:01:30.122909Z 2021-12-17T14:01:09.028254Z 2021-12-17T14:01:39.078548Z 2021-12-17T14:01:02.919606Z 2021-12-17T14:01:29.862726Z 2021-12-17T14:01:05.792852Z 2021-12-17T14:01:36.637139Z 2021-12-17T14:01:06.903058Z 2021-12-17T14:01:29.720542Z 2021-12-17T14:00:50.667272Z 2021-12-17T14:01:40.085347Z 2021-12-17T14:00:57.462793Z 2021-12-17T14:01:42.695241Z 2021-12-17T14:07:24.500184Z 2021-12-17T14:07:14.171247Z 2021-12-17T14:07:23.373232Z 2021-12-17T14:07:14.604168Z 2021-12-17T14:07:21.605433Z 2021-12-17T14:07:12.419387Z 2021-12-17T14:07:23.552073Z 2021-12-17T14:07:13.187339Z 2021-12-17T14:07:24.343000Z 2021-12-17T14:07:12.598679Z 2021-12-17T14:07:22.512653Z 2021-12-17T14:07:12.721038Z 2021-12-17T14:07:20.447098Z 2021-12-17T14:07:12.481676Z 2021-12-17T14:07:17.804215Z 2021-12-17T14:07:10.909409Z 2021-12-17T14:07:18.349708Z 2021-12-17T14:07:21.541978Z 2021-12-17T14:07:08.577483Z 2021-12-17T14:07:10.897380Z 2021-12-17T14:07:19.612651Z 2021-12-17T14:07:12.670189Z 2021-12-17T14:07:14.514978Z 2021-12-17T14:07:11.321303Z 2021-12-17T14:07:22.272210Z 2021-12-17T14:07:09.277737Z 2021-12-17T14:07:12.100603Z 2021-12-17T14:07:06.499663Z 2021-12-17T14:07:19.246591Z 2021-12-17T14:07:07.308237Z 2021-12-17T14:07:18.139182Z 2021-12-17T14:07:10.323120Z 2021-12-17T14:07:19.613069Z 2021-12-17T14:07:06.862079Z 2021-12-17T14:07:19.660759Z 2021-12-17T14:07:02.716448Z 2021-12-17T14:07:20.788600Z 2021-12-17T14:07:05.433840Z 2021-12-17T14:07:17.695724Z 2021-12-17T14:07:00.834509Z 2021-12-17T14:07:11.800753Z 2021-12-17T14:06:57.875017Z 2021-12-17T14:07:11.462236Z 2021-12-17T14:06:49.851479Z 2021-12-17T14:07:06.335110Z 2021-12-17T14:06:52.797065Z 2021-12-17T14:07:09.265288Z 2021-12-17T14:06:54.750006Z 2021-12-17T14:07:06.822387Z 2021-12-17T14:06:58.579110Z 2021-12-17T14:07:13.064863Z 2021-12-17T14:06:49.245697Z 2021-12-17T14:07:01.108926Z 2021-12-17T14:06:55.355176Z 2021-12-17T14:07:09.408841Z 2021-12-17T14:06:48.332187Z 2021-12-17T14:07:09.673266Z 2021-12-17T14:06:49.204114Z 2021-12-17T14:07:09.580272Z 2021-12-17T14:06:43.297687Z 2021-12-17T14:07:03.562247Z 2021-12-17T14:06:43.026247Z 2021-12-17T14:06:58.175181Z 2021-12-17T14:06:35.371487Z 2021-12-17T14:06:58.256318Z 2021-12-17T14:06:27.922377Z 2021-12-17T14:06:55.000669Z 2021-12-17T14:06:34.022809Z 2021-12-17T14:05:57.768842Z 2021-12-17T14:06:20.584648Z 2021-12-17T14:05:55.959017Z 2021-12-17T14:06:26.225522Z 2021-12-17T14:05:55.330369Z 2021-12-17T14:06:12.173124Z 2021-12-17T14:05:54.647276Z 2021-12-17T14:06:21.489242Z 2021-12-17T14:05:52.820432Z 2021-12-17T14:06:19.912292Z 2021-12-17T14:05:51.322943Z 2021-12-17T14:06:09.586177Z 2021-12-17T14:05:49.960678Z 2021-12-17T14:06:09.892255Z 2021-12-17T14:05:48.773218Z 2021-12-17T14:06:02.330177Z 2021-12-17T14:05:45.444041Z 2021-12-17T14:05:53.962079Z 2021-12-17T14:05:42.852995Z 2021-12-17T14:05:58.668845Z 2021-12-17T14:05:43.316260Z 2021-12-17T14:06:01.700636Z 2021-12-17T14:05:32.488076Z 2021-12-17T14:05:39.558668Z 2021-12-17T14:05:33.161626Z 2021-12-17T14:05:38.060299Z 2021-12-17T14:05:32.876642Z 2021-12-17T14:05:38.110261Z 2021-12-17T14:05:31.133268Z 2021-12-17T14:05:32.972821Z 2021-12-17T14:05:28.530144Z 2021-12-17T14:05:24.207436Z 2021-12-17T14:12:03.914770Z 2021-12-17T14:12:11.617387Z 2021-12-17T14:11:59.177242Z 2021-12-17T14:12:13.985180Z 2021-12-17T14:12:03.913403Z 2021-12-17T14:12:14.083486Z 2021-12-17T14:12:03.668846Z 2021-12-17T14:12:11.774674Z 2021-12-17T14:12:02.888927Z 2021-12-17T14:12:13.685516Z 2021-12-17T14:12:04.149767Z 2021-12-17T14:12:12.019498Z 2021-12-17T14:12:02.285857Z 2021-12-17T14:12:08.036295Z 2021-12-17T14:12:02.104088Z 2021-12-17T14:12:09.407392Z 2021-12-17T14:12:02.386492Z 2021-12-17T14:12:08.045911Z 2021-12-17T14:12:01.451763Z 2021-12-17T14:12:13.991619Z 2021-12-17T14:12:02.635610Z 2021-12-17T14:12:09.891191Z 2021-12-17T14:11:55.667137Z 2021-12-17T14:12:10.814397Z 2021-12-17T14:12:00.831345Z 2021-12-17T14:12:13.536191Z 2021-12-17T14:12:03.510477Z 2021-12-17T14:12:11.083791Z 2021-12-17T14:11:58.238365Z 2021-12-17T14:12:11.708901Z 2021-12-17T14:11:58.199237Z 2021-12-17T14:12:06.351487Z 2021-12-17T14:11:58.603855Z 2021-12-17T14:12:07.224613Z 2021-12-17T14:11:57.426138Z 2021-12-17T14:12:11.599175Z 2021-12-17T14:12:00.262132Z 2021-12-17T14:12:08.365454Z 2021-12-17T14:11:51.756208Z 2021-12-17T14:12:02.521264Z 2021-12-17T14:11:52.229979Z 2021-12-17T14:11:57.197250Z 2021-12-17T14:11:56.418010Z 2021-12-17T14:12:12.139638Z 2021-12-17T14:11:50.755012Z 2021-12-17T14:12:02.820350Z 2021-12-17T14:11:51.061725Z 2021-12-17T14:12:08.216246Z 2021-12-17T14:11:54.231272Z 2021-12-17T14:12:03.243278Z 2021-12-17T14:11:52.130287Z 2021-12-17T14:11:39.536173Z 2021-12-17T14:11:50.126500Z 2021-12-17T14:12:03.559134Z 2021-12-17T14:11:45.318297Z 2021-12-17T14:11:47.263274Z 2021-12-17T14:11:38.168423Z 2021-12-17T14:11:54.336850Z 2021-12-17T14:11:33.057978Z 2021-12-17T14:11:43.371081Z 2021-12-17T14:11:31.301147Z 2021-12-17T14:11:53.873230Z 2021-12-17T14:11:37.992009Z 2021-12-17T14:11:46.064995Z 2021-12-17T14:11:35.816462Z 2021-12-17T14:11:47.021973Z 2021-12-17T14:11:31.511421Z 2021-12-17T14:11:42.924236Z 2021-12-17T14:11:35.498101Z 2021-12-17T14:11:25.576249Z 2021-12-17T14:11:36.229082Z 2021-12-17T14:11:11.526974Z 2021-12-17T14:11:34.920998Z 2021-12-17T14:11:25.504974Z 2021-12-17T14:11:27.892833Z 2021-12-17T14:11:24.478655Z 2021-12-17T14:11:20.532759Z 2021-12-17T14:11:18.239958Z 2021-12-17T14:11:24.981270Z 2021-12-17T14:11:34.281269Z 2021-12-17T14:11:21.634543Z 2021-12-17T14:11:27.661814Z 2021-12-17T14:11:08.981007Z 2021-12-17T14:11:23.700499Z 2021-12-17T14:11:04.149465Z 2021-12-17T14:11:12.260642Z 2021-12-17T14:11:03.571200Z 2021-12-17T14:11:08.633244Z 2021-12-17T14:11:00.254934Z 2021-12-17T14:11:00.272158Z 2021-12-17T14:10:24.600202Z 2021-12-17T14:10:25.168483Z 2021-12-17T14:10:21.029456Z 2021-12-17T14:10:23.342743Z 2021-12-17T14:10:19.422497Z 2021-12-17T14:10:21.042774Z 2021-12-17T14:10:13.360927Z 2021-12-17T14:10:13.255193Z 2021-12-17T14:10:08.911454Z 2021-12-17T14:10:09.477050Z 2021-12-17T14:17:03.059683Z 2021-12-17T14:17:06.447047Z 2021-12-17T14:16:57.922385Z 2021-12-17T14:17:07.093451Z 2021-12-17T14:17:02.174005Z 2021-12-17T14:17:06.480553Z 2021-12-17T14:16:58.017698Z 2021-12-17T14:17:00.129025Z 2021-12-17T14:17:00.240423Z 2021-12-17T14:17:07.535258Z 2021-12-17T14:17:02.139693Z 2021-12-17T14:17:06.570205Z 2021-12-17T14:17:03.224987Z 2021-12-17T14:17:08.033267Z 2021-12-17T14:16:57.865178Z 2021-12-17T14:17:03.971920Z 2021-12-17T14:17:00.625420Z 2021-12-17T14:17:08.015999Z 2021-12-17T14:16:59.376375Z 2021-12-17T14:17:00.430145Z 2021-12-17T14:17:01.584001Z 2021-12-17T14:17:07.922126Z 2021-12-17T14:17:01.619732Z 2021-12-17T14:17:07.462412Z 2021-12-17T14:16:56.336542Z 2021-12-17T14:17:02.933773Z 2021-12-17T14:17:02.061661Z 2021-12-17T14:16:59.215299Z 2021-12-17T14:16:59.120892Z 2021-12-17T14:16:55.509804Z 2021-12-17T14:16:56.137191Z 2021-12-17T14:16:57.418137Z 2021-12-17T14:17:01.798328Z 2021-12-17T14:16:57.299912Z 2021-12-17T14:16:56.075284Z 2021-12-17T14:16:57.440218Z 2021-12-17T14:16:45.698193Z 2021-12-17T14:16:56.722652Z 2021-12-17T14:16:53.301391Z 2021-12-17T14:17:04.233455Z 2021-12-17T14:16:50.414127Z 2021-12-17T14:17:01.657105Z 2021-12-17T14:16:49.483115Z 2021-12-17T14:16:46.706256Z 2021-12-17T14:16:50.467372Z 2021-12-17T14:16:55.204897Z 2021-12-17T14:16:53.829956Z 2021-12-17T14:16:53.645715Z 2021-12-17T14:16:55.064811Z 2021-12-17T14:17:04.586426Z 2021-12-17T14:16:48.194098Z 2021-12-17T14:16:56.220886Z 2021-12-17T14:16:49.781780Z 2021-12-17T14:16:50.967317Z 2021-12-17T14:16:47.427033Z 2021-12-17T14:16:48.624400Z 2021-12-17T14:16:52.850698Z 2021-12-17T14:16:54.524246Z 2021-12-17T14:16:47.905361Z 2021-12-17T14:16:45.280097Z 2021-12-17T14:16:36.048370Z 2021-12-17T14:16:45.333573Z 2021-12-17T14:16:38.586707Z 2021-12-17T14:16:48.832547Z 2021-12-17T14:16:36.023119Z 2021-12-17T14:16:16.395737Z 2021-12-17T14:16:10.127394Z 2021-12-17T14:16:30.639185Z 2021-12-17T14:16:22.309043Z 2021-12-17T14:16:27.433206Z 2021-12-17T14:16:26.745069Z 2021-12-17T14:16:31.146363Z 2021-12-17T14:16:05.633007Z 2021-12-17T14:16:19.582554Z 2021-12-17T14:16:13.735181Z 2021-12-17T14:16:24.027995Z 2021-12-17T14:16:19.309957Z 2021-12-17T14:16:18.525280Z 2021-12-17T14:16:16.208096Z 2021-12-17T14:16:16.315760Z 2021-12-17T14:16:07.332748Z 2021-12-17T14:16:17.569497Z 2021-12-17T14:16:01.685819Z 2021-12-17T14:16:07.304630Z 2021-12-17T14:16:07.295942Z 2021-12-17T14:16:07.394088Z 2021-12-17T14:16:06.988120Z 2021-12-17T14:15:53.067222Z 2021-12-17T14:16:00.524144Z 2021-12-17T14:15:52.060210Z 2021-12-17T14:16:07.028040Z 2021-12-17T14:15:38.995400Z 2021-12-17T14:15:41.434392Z 2021-12-17T14:15:33.341340Z 2021-12-17T14:15:41.525917Z 2021-12-17T14:15:41.530454Z 2021-12-17T14:15:16.015818Z 2021-12-17T14:15:16.763009Z 2021-12-17T14:15:06.109763Z 2021-12-17T14:15:00.934360Z 2021-12-17T14:22:03.076696Z 2021-12-17T14:21:47.759098Z 2021-12-17T14:22:05.439815Z 2021-12-17T14:21:47.462729Z 2021-12-17T14:22:04.855861Z 2021-12-17T14:21:46.270453Z 2021-12-17T14:22:05.112270Z 2021-12-17T14:21:46.871113Z 2021-12-17T14:22:05.795000Z 2021-12-17T14:21:46.024029Z 2021-12-17T14:22:03.156111Z 2021-12-17T14:21:55.240183Z 2021-12-17T14:21:48.001831Z 2021-12-17T14:21:40.824216Z 2021-12-17T14:21:55.645704Z 2021-12-17T14:21:34.946159Z 2021-12-17T14:21:58.357035Z 2021-12-17T14:21:40.842056Z 2021-12-17T14:22:00.189395Z 2021-12-17T14:22:01.994095Z 2021-12-17T14:21:40.169207Z 2021-12-17T14:22:00.792274Z 2021-12-17T14:21:37.192381Z 2021-12-17T14:22:00.104066Z 2021-12-17T14:21:42.855343Z 2021-12-17T14:22:00.197308Z 2021-12-17T14:21:42.357465Z 2021-12-17T14:21:59.818063Z 2021-12-17T14:21:40.048118Z 2021-12-17T14:21:56.446729Z 2021-12-17T14:21:41.750766Z 2021-12-17T14:21:45.739457Z 2021-12-17T14:21:45.192148Z 2021-12-17T14:21:38.369332Z 2021-12-17T14:21:56.406276Z 2021-12-17T14:21:37.364768Z 2021-12-17T14:21:49.428054Z 2021-12-17T14:21:31.768223Z 2021-12-17T14:22:01.139551Z 2021-12-17T14:22:05.264682Z 2021-12-17T14:21:39.317062Z 2021-12-17T14:21:57.158458Z 2021-12-17T14:21:38.125734Z 2021-12-17T14:21:52.301812Z 2021-12-17T14:21:42.703416Z 2021-12-17T14:21:52.668492Z 2021-12-17T14:21:35.299014Z 2021-12-17T14:21:54.440125Z 2021-12-17T14:21:37.163576Z 2021-12-17T14:21:50.863336Z 2021-12-17T14:21:35.224848Z 2021-12-17T14:21:51.529856Z 2021-12-17T14:21:14.552172Z 2021-12-17T14:21:42.273054Z 2021-12-17T14:21:29.173169Z 2021-12-17T14:21:43.500902Z 2021-12-17T14:21:25.843101Z 2021-12-17T14:21:40.398628Z 2021-12-17T14:21:29.144257Z 2021-12-17T14:21:40.129890Z 2021-12-17T14:21:25.301123Z 2021-12-17T14:21:39.573379Z 2021-12-17T14:21:22.700115Z 2021-12-17T14:21:40.585734Z 2021-12-17T14:21:16.999011Z 2021-12-17T14:21:40.654382Z 2021-12-17T14:21:17.094905Z 2021-12-17T14:21:36.293789Z 2021-12-17T14:21:13.654839Z 2021-12-17T14:21:35.028319Z 2021-12-17T14:21:22.899767Z 2021-12-17T14:21:25.483158Z 2021-12-17T14:21:12.523995Z 2021-12-17T14:21:28.042287Z 2021-12-17T14:21:21.636500Z 2021-12-17T14:21:23.606199Z 2021-12-17T14:21:03.500381Z 2021-12-17T14:21:21.200473Z 2021-12-17T14:21:00.878703Z 2021-12-17T14:21:11.342278Z 2021-12-17T14:21:09.273298Z 2021-12-17T14:21:05.861397Z 2021-12-17T14:20:59.416694Z 2021-12-17T14:21:18.432238Z 2021-12-17T14:20:50.608718Z 2021-12-17T14:21:12.087399Z 2021-12-17T14:20:47.358827Z 2021-12-17T14:21:00.280134Z 2021-12-17T14:20:45.043907Z 2021-12-17T14:20:09.562796Z 2021-12-17T14:20:03.239408Z 2021-12-17T14:20:18.222855Z 2021-12-17T14:20:04.123284Z 2021-12-17T14:20:11.847133Z 2021-12-17T14:19:59.149038Z 2021-12-17T14:20:11.072836Z 2021-12-17T14:20:02.420279Z 2021-12-17T14:19:56.758611Z 2021-12-17T14:19:51.616887Z 2021-12-17T14:19:51.170595Z", + "finished_diff": 138.8721032, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 191.616398, + "mean": 70.96987857799995, + "min": 12.918322 + }, + "jobs_events_duration": { + "max": 10.224817, + "mean": 4.4170425359999985, + "min": 0.177238 + }, + "jobs_events_lag": { + "max": -1.657826, + "mean": -5.4988084, + "min": -9.802088 + }, + "jobs_waiting": { + "max": 59.553451, + "mean": 26.58376873199998, + "min": 0.604175 + }, + "modified": "2021-12-17T14:00:02.337109Z 2021-12-17T14:00:02.302598Z 2021-12-17T14:00:02.269048Z 2021-12-17T14:00:02.232762Z 2021-12-17T14:00:02.197793Z 2021-12-17T14:00:02.161194Z 2021-12-17T14:00:02.126526Z 2021-12-17T13:59:55.043489Z 2021-12-17T13:59:54.870481Z 2021-12-17T13:59:54.813349Z 2021-12-17T13:59:54.703102Z 2021-12-17T13:59:54.647568Z 2021-12-17T13:59:54.594586Z 2021-12-17T13:59:54.543981Z 2021-12-17T13:59:54.444451Z 2021-12-17T13:59:54.236237Z 2021-12-17T13:59:54.152397Z 2021-12-17T13:59:54.047563Z 2021-12-17T13:59:53.986476Z 2021-12-17T13:59:53.923017Z 2021-12-17T13:59:53.784027Z 2021-12-17T13:59:53.615130Z 2021-12-17T13:59:53.535937Z 2021-12-17T13:59:53.429183Z 2021-12-17T13:59:53.382347Z 2021-12-17T13:59:53.233484Z 2021-12-17T13:59:53.178966Z 2021-12-17T13:59:53.084262Z 2021-12-17T13:59:50.524163Z 2021-12-17T13:59:50.449797Z 2021-12-17T13:59:50.407265Z 2021-12-17T13:59:50.273644Z 2021-12-17T13:59:50.199391Z 2021-12-17T13:59:49.306053Z 2021-12-17T13:59:49.248535Z 2021-12-17T13:59:47.855041Z 2021-12-17T14:00:31.950329Z 2021-12-17T14:00:31.720735Z 2021-12-17T14:00:31.521110Z 2021-12-17T14:00:31.362656Z 2021-12-17T14:00:22.883040Z 2021-12-17T14:00:22.761118Z 2021-12-17T14:00:22.641088Z 2021-12-17T14:00:22.540046Z 2021-12-17T14:00:22.451035Z 2021-12-17T14:00:22.380006Z 2021-12-17T14:00:22.299285Z 2021-12-17T14:00:22.177016Z 2021-12-17T14:00:22.061045Z 2021-12-17T14:00:21.957422Z 2021-12-17T14:00:21.912349Z 2021-12-17T14:00:21.823926Z 2021-12-17T14:00:21.717966Z 2021-12-17T14:00:21.616238Z 2021-12-17T14:00:21.551052Z 2021-12-17T14:00:21.459300Z 2021-12-17T14:00:04.054167Z 2021-12-17T14:00:04.021659Z 2021-12-17T14:00:03.989999Z 2021-12-17T14:00:21.350358Z 2021-12-17T14:00:03.953212Z 2021-12-17T14:00:03.917833Z 2021-12-17T14:00:03.882001Z 2021-12-17T14:00:03.849055Z 2021-12-17T14:00:03.816023Z 2021-12-17T14:00:03.778206Z 2021-12-17T14:00:03.743270Z 2021-12-17T14:00:03.707273Z 2021-12-17T14:00:03.672827Z 2021-12-17T14:00:03.633339Z 2021-12-17T14:00:03.592784Z 2021-12-17T14:00:03.550516Z 2021-12-17T14:00:03.503551Z 2021-12-17T14:00:03.465099Z 2021-12-17T14:00:03.417997Z 2021-12-17T14:00:03.354252Z 2021-12-17T14:00:03.313070Z 2021-12-17T14:00:03.277443Z 2021-12-17T14:00:03.240179Z 2021-12-17T14:00:03.204064Z 2021-12-17T14:00:03.169975Z 2021-12-17T14:00:03.088955Z 2021-12-17T14:00:03.030412Z 2021-12-17T14:00:02.983576Z 2021-12-17T14:00:02.926075Z 2021-12-17T14:00:02.875029Z 2021-12-17T14:00:02.836899Z 2021-12-17T14:00:02.798648Z 2021-12-17T14:00:02.765508Z 2021-12-17T14:00:02.724273Z 2021-12-17T14:00:02.684081Z 2021-12-17T14:00:02.650419Z 2021-12-17T14:00:02.609362Z 2021-12-17T14:00:02.575061Z 2021-12-17T14:00:02.540645Z 2021-12-17T14:00:02.505792Z 2021-12-17T14:00:02.470334Z 2021-12-17T14:00:02.437507Z 2021-12-17T14:00:02.404608Z 2021-12-17T14:00:02.371530Z 2021-12-17T14:05:46.425531Z 2021-12-17T14:05:46.315030Z 2021-12-17T14:05:46.175139Z 2021-12-17T14:05:46.085723Z 2021-12-17T14:05:45.997257Z 2021-12-17T14:05:45.901342Z 2021-12-17T14:05:45.832488Z 2021-12-17T14:05:45.759045Z 2021-12-17T14:05:45.692105Z 2021-12-17T14:05:45.634292Z 2021-12-17T14:05:45.554120Z 2021-12-17T14:05:45.467470Z 2021-12-17T14:05:45.404381Z 2021-12-17T14:05:45.341049Z 2021-12-17T14:05:45.280533Z 2021-12-17T14:05:45.213129Z 2021-12-17T14:05:45.148163Z 2021-12-17T14:05:44.981321Z 2021-12-17T14:05:45.072384Z 2021-12-17T14:05:44.890222Z 2021-12-17T14:05:44.799111Z 2021-12-17T14:05:44.677350Z 2021-12-17T14:05:44.577193Z 2021-12-17T14:05:44.488196Z 2021-12-17T14:05:44.399357Z 2021-12-17T14:05:44.313125Z 2021-12-17T14:05:44.206261Z 2021-12-17T14:05:44.100422Z 2021-12-17T14:05:44.053361Z 2021-12-17T14:05:43.915231Z 2021-12-17T14:05:43.763282Z 2021-12-17T14:05:43.638244Z 2021-12-17T14:05:43.553268Z 2021-12-17T14:05:43.468338Z 2021-12-17T14:05:43.384369Z 2021-12-17T14:05:43.299334Z 2021-12-17T14:05:43.206261Z 2021-12-17T14:05:43.118269Z 2021-12-17T14:05:43.018156Z 2021-12-17T14:05:42.913101Z 2021-12-17T14:05:42.814264Z 2021-12-17T14:05:42.714262Z 2021-12-17T14:05:42.613137Z 2021-12-17T14:05:42.527334Z 2021-12-17T14:05:42.437188Z 2021-12-17T14:05:42.355972Z 2021-12-17T14:05:42.293215Z 2021-12-17T14:05:42.230129Z 2021-12-17T14:05:42.166274Z 2021-12-17T14:05:42.106608Z 2021-12-17T14:05:42.043152Z 2021-12-17T14:05:42.000078Z 2021-12-17T14:05:41.959168Z 2021-12-17T14:05:41.912237Z 2021-12-17T14:05:41.868541Z 2021-12-17T14:05:41.813185Z 2021-12-17T14:05:41.759324Z 2021-12-17T14:05:41.686351Z 2021-12-17T14:05:41.612244Z 2021-12-17T14:05:41.553195Z 2021-12-17T14:05:41.487324Z 2021-12-17T14:05:41.423161Z 2021-12-17T14:05:41.357241Z 2021-12-17T14:05:41.288341Z 2021-12-17T14:05:41.208343Z 2021-12-17T14:05:41.139355Z 2021-12-17T14:05:41.046850Z 2021-12-17T14:05:40.950470Z 2021-12-17T14:05:25.550336Z 2021-12-17T14:05:25.123698Z 2021-12-17T14:05:24.538305Z 2021-12-17T14:05:23.988164Z 2021-12-17T14:05:23.651092Z 2021-12-17T14:05:23.176562Z 2021-12-17T14:05:22.760461Z 2021-12-17T14:05:22.344551Z 2021-12-17T14:05:21.756419Z 2021-12-17T14:05:21.328327Z 2021-12-17T14:05:20.912485Z 2021-12-17T14:05:20.552624Z 2021-12-17T14:05:20.353577Z 2021-12-17T14:05:20.139515Z 2021-12-17T14:05:19.954307Z 2021-12-17T14:05:19.355099Z 2021-12-17T14:05:18.619247Z 2021-12-17T14:05:18.253928Z 2021-12-17T14:05:18.030854Z 2021-12-17T14:05:17.654805Z 2021-12-17T14:05:17.249496Z 2021-12-17T14:05:16.843313Z 2021-12-17T14:05:11.806253Z 2021-12-17T14:05:11.703638Z 2021-12-17T14:05:11.615679Z 2021-12-17T14:05:11.499922Z 2021-12-17T14:05:11.448647Z 2021-12-17T14:05:11.394067Z 2021-12-17T14:05:11.296738Z 2021-12-17T14:05:10.233190Z 2021-12-17T14:05:10.176669Z 2021-12-17T14:05:09.127625Z 2021-12-17T14:10:32.630192Z 2021-12-17T14:10:32.558600Z 2021-12-17T14:10:32.463297Z 2021-12-17T14:10:32.395399Z 2021-12-17T14:10:32.289642Z 2021-12-17T14:10:32.078155Z 2021-12-17T14:10:31.961034Z 2021-12-17T14:10:31.804527Z 2021-12-17T14:10:31.650542Z 2021-12-17T14:10:31.582721Z 2021-12-17T14:10:31.518312Z 2021-12-17T14:10:31.457756Z 2021-12-17T14:10:31.415444Z 2021-12-17T14:10:31.358849Z 2021-12-17T14:10:31.299268Z 2021-12-17T14:10:31.228289Z 2021-12-17T14:10:31.167294Z 2021-12-17T14:10:31.103288Z 2021-12-17T14:10:31.048475Z 2021-12-17T14:10:30.994639Z 2021-12-17T14:10:30.937386Z 2021-12-17T14:10:30.868620Z 2021-12-17T14:10:30.782456Z 2021-12-17T14:10:30.703384Z 2021-12-17T14:10:30.643379Z 2021-12-17T14:10:30.579326Z 2021-12-17T14:10:30.505554Z 2021-12-17T14:10:30.423550Z 2021-12-17T14:10:30.353310Z 2021-12-17T14:10:30.285350Z 2021-12-17T14:10:30.190668Z 2021-12-17T14:10:30.063421Z 2021-12-17T14:10:29.964579Z 2021-12-17T14:10:29.877324Z 2021-12-17T14:10:29.785377Z 2021-12-17T14:10:29.670438Z 2021-12-17T14:10:29.569355Z 2021-12-17T14:10:29.431360Z 2021-12-17T14:10:29.256253Z 2021-12-17T14:10:29.102791Z 2021-12-17T14:10:28.949573Z 2021-12-17T14:10:28.812464Z 2021-12-17T14:10:28.678344Z 2021-12-17T14:10:28.573344Z 2021-12-17T14:10:28.470414Z 2021-12-17T14:10:28.368691Z 2021-12-17T14:10:28.260920Z 2021-12-17T14:10:28.138580Z 2021-12-17T14:10:27.942350Z 2021-12-17T14:10:27.832322Z 2021-12-17T14:10:27.725911Z 2021-12-17T14:10:11.469099Z 2021-12-17T14:10:11.424470Z 2021-12-17T14:10:27.573812Z 2021-12-17T14:10:11.386372Z 2021-12-17T14:10:11.348816Z 2021-12-17T14:10:11.299042Z 2021-12-17T14:10:11.259558Z 2021-12-17T14:10:11.187059Z 2021-12-17T14:10:11.106325Z 2021-12-17T14:10:11.048007Z 2021-12-17T14:10:11.012914Z 2021-12-17T14:10:10.977457Z 2021-12-17T14:10:10.940554Z 2021-12-17T14:10:10.896327Z 2021-12-17T14:10:10.856472Z 2021-12-17T14:10:10.816879Z 2021-12-17T14:10:10.772793Z 2021-12-17T14:10:10.652380Z 2021-12-17T14:10:10.512410Z 2021-12-17T14:10:10.139177Z 2021-12-17T14:10:10.275292Z 2021-12-17T14:10:10.073378Z 2021-12-17T14:10:09.971567Z 2021-12-17T14:10:09.803516Z 2021-12-17T14:10:09.689574Z 2021-12-17T14:10:09.574147Z 2021-12-17T14:10:09.470516Z 2021-12-17T14:10:09.342932Z 2021-12-17T14:10:09.227443Z 2021-12-17T14:10:09.103598Z 2021-12-17T14:10:08.848362Z 2021-12-17T14:10:08.540833Z 2021-12-17T14:10:08.358103Z 2021-12-17T14:10:08.168802Z 2021-12-17T14:10:07.986499Z 2021-12-17T14:10:07.787323Z 2021-12-17T14:10:07.426069Z 2021-12-17T14:10:06.921209Z 2021-12-17T14:10:06.431762Z 2021-12-17T14:09:56.782645Z 2021-12-17T14:09:56.559367Z 2021-12-17T14:09:56.412590Z 2021-12-17T14:09:56.275108Z 2021-12-17T14:09:55.982693Z 2021-12-17T14:09:55.792440Z 2021-12-17T14:09:54.391728Z 2021-12-17T14:09:53.556342Z 2021-12-17T14:09:53.519816Z 2021-12-17T14:09:52.909123Z 2021-12-17T14:15:16.981069Z 2021-12-17T14:15:16.894317Z 2021-12-17T14:15:16.780302Z 2021-12-17T14:15:16.656334Z 2021-12-17T14:15:16.484320Z 2021-12-17T14:15:16.306205Z 2021-12-17T14:15:16.172994Z 2021-12-17T14:15:16.111097Z 2021-12-17T14:15:16.030654Z 2021-12-17T14:15:15.951284Z 2021-12-17T14:15:15.899283Z 2021-12-17T14:15:15.836071Z 2021-12-17T14:15:15.775216Z 2021-12-17T14:15:15.717044Z 2021-12-17T14:15:15.659275Z 2021-12-17T14:15:15.585117Z 2021-12-17T14:15:15.481443Z 2021-12-17T14:15:15.390141Z 2021-12-17T14:15:15.313119Z 2021-12-17T14:15:15.226342Z 2021-12-17T14:15:15.135205Z 2021-12-17T14:15:15.024375Z 2021-12-17T14:15:14.932210Z 2021-12-17T14:15:14.846224Z 2021-12-17T14:15:14.745304Z 2021-12-17T14:15:14.664606Z 2021-12-17T14:15:14.564423Z 2021-12-17T14:15:14.440413Z 2021-12-17T14:15:14.305581Z 2021-12-17T14:15:14.164259Z 2021-12-17T14:15:14.045837Z 2021-12-17T14:15:13.956607Z 2021-12-17T14:15:13.850361Z 2021-12-17T14:15:13.734467Z 2021-12-17T14:15:13.617472Z 2021-12-17T14:15:13.494292Z 2021-12-17T14:15:13.393440Z 2021-12-17T14:15:13.299158Z 2021-12-17T14:15:13.211248Z 2021-12-17T14:15:13.132560Z 2021-12-17T14:15:13.040242Z 2021-12-17T14:15:12.942119Z 2021-12-17T14:15:12.860176Z 2021-12-17T14:15:12.778241Z 2021-12-17T14:15:12.685587Z 2021-12-17T14:15:12.546480Z 2021-12-17T14:15:12.340647Z 2021-12-17T14:15:12.224368Z 2021-12-17T14:15:12.102434Z 2021-12-17T14:15:12.028240Z 2021-12-17T14:15:11.954627Z 2021-12-17T14:15:11.869215Z 2021-12-17T14:15:11.779210Z 2021-12-17T14:15:11.640009Z 2021-12-17T14:15:11.536479Z 2021-12-17T14:15:11.397333Z 2021-12-17T14:15:11.283207Z 2021-12-17T14:15:11.209701Z 2021-12-17T14:15:11.154211Z 2021-12-17T14:15:11.100062Z 2021-12-17T14:15:11.045282Z 2021-12-17T14:15:10.992842Z 2021-12-17T14:15:10.904093Z 2021-12-17T14:15:10.806395Z 2021-12-17T14:15:10.725491Z 2021-12-17T14:14:58.118408Z 2021-12-17T14:14:58.077001Z 2021-12-17T14:14:58.037787Z 2021-12-17T14:14:57.999157Z 2021-12-17T14:14:57.961589Z 2021-12-17T14:14:57.918639Z 2021-12-17T14:14:57.875434Z 2021-12-17T14:14:57.793281Z 2021-12-17T14:14:57.672221Z 2021-12-17T14:14:57.594141Z 2021-12-17T14:14:57.502318Z 2021-12-17T14:14:57.400165Z 2021-12-17T14:14:57.307091Z 2021-12-17T14:14:57.157366Z 2021-12-17T14:14:57.095547Z 2021-12-17T14:14:56.967974Z 2021-12-17T14:14:56.878858Z 2021-12-17T14:14:56.747568Z 2021-12-17T14:14:56.594708Z 2021-12-17T14:14:56.308373Z 2021-12-17T14:14:55.973021Z 2021-12-17T14:14:55.585376Z 2021-12-17T14:14:55.196133Z 2021-12-17T14:14:55.006631Z 2021-12-17T14:14:54.711525Z 2021-12-17T14:14:54.238393Z 2021-12-17T14:14:53.726683Z 2021-12-17T14:14:53.223560Z 2021-12-17T14:14:52.919638Z 2021-12-17T14:14:52.586026Z 2021-12-17T14:14:52.235259Z 2021-12-17T14:14:48.137750Z 2021-12-17T14:14:47.912520Z 2021-12-17T14:14:47.710427Z 2021-12-17T14:14:46.179575Z 2021-12-17T14:20:40.933336Z 2021-12-17T14:20:40.731402Z 2021-12-17T14:20:40.658146Z 2021-12-17T14:20:40.612035Z 2021-12-17T14:20:40.562018Z 2021-12-17T14:20:40.517028Z 2021-12-17T14:20:40.451147Z 2021-12-17T14:20:40.376049Z 2021-12-17T14:20:40.308459Z 2021-12-17T14:20:40.257322Z 2021-12-17T14:20:40.145221Z 2021-12-17T14:19:53.101741Z 2021-12-17T14:20:40.015224Z 2021-12-17T14:19:53.061189Z 2021-12-17T14:19:53.001415Z 2021-12-17T14:19:52.948371Z 2021-12-17T14:19:52.895559Z 2021-12-17T14:19:52.854296Z 2021-12-17T14:19:52.816898Z 2021-12-17T14:20:39.841558Z 2021-12-17T14:19:52.774851Z 2021-12-17T14:19:52.730752Z 2021-12-17T14:19:52.693033Z 2021-12-17T14:19:52.655546Z 2021-12-17T14:19:52.616605Z 2021-12-17T14:19:52.571102Z 2021-12-17T14:19:52.524078Z 2021-12-17T14:19:52.477413Z 2021-12-17T14:19:52.436517Z 2021-12-17T14:19:52.389723Z 2021-12-17T14:19:52.348824Z 2021-12-17T14:19:52.307916Z 2021-12-17T14:20:39.673145Z 2021-12-17T14:19:52.261044Z 2021-12-17T14:19:52.221090Z 2021-12-17T14:19:52.181373Z 2021-12-17T14:19:52.112764Z 2021-12-17T14:19:52.068802Z 2021-12-17T14:19:52.020280Z 2021-12-17T14:20:39.564608Z 2021-12-17T14:19:51.968640Z 2021-12-17T14:19:51.908805Z 2021-12-17T14:19:51.825183Z 2021-12-17T14:19:51.738111Z 2021-12-17T14:19:51.626631Z 2021-12-17T14:19:51.554729Z 2021-12-17T14:19:51.492337Z 2021-12-17T14:19:51.449952Z 2021-12-17T14:19:51.408118Z 2021-12-17T14:19:51.371003Z 2021-12-17T14:19:51.326597Z 2021-12-17T14:19:51.284765Z 2021-12-17T14:19:51.247351Z 2021-12-17T14:19:51.208245Z 2021-12-17T14:19:51.171080Z 2021-12-17T14:19:51.124961Z 2021-12-17T14:19:51.071747Z 2021-12-17T14:19:51.017426Z 2021-12-17T14:19:50.967353Z 2021-12-17T14:19:50.906236Z 2021-12-17T14:19:50.844806Z 2021-12-17T14:19:50.789498Z 2021-12-17T14:19:50.716810Z 2021-12-17T14:19:50.646769Z 2021-12-17T14:19:50.599524Z 2021-12-17T14:19:50.539376Z 2021-12-17T14:19:50.495820Z 2021-12-17T14:19:50.454125Z 2021-12-17T14:19:50.400429Z 2021-12-17T14:19:50.351286Z 2021-12-17T14:19:50.283323Z 2021-12-17T14:19:50.239613Z 2021-12-17T14:19:50.137953Z 2021-12-17T14:19:50.074636Z 2021-12-17T14:19:50.003293Z 2021-12-17T14:19:49.938804Z 2021-12-17T14:19:49.868188Z 2021-12-17T14:19:49.789511Z 2021-12-17T14:19:49.718792Z 2021-12-17T14:19:49.660099Z 2021-12-17T14:19:49.603765Z 2021-12-17T14:19:49.564799Z 2021-12-17T14:19:49.503672Z 2021-12-17T14:19:49.449012Z 2021-12-17T14:19:49.395129Z 2021-12-17T14:19:49.324579Z 2021-12-17T14:19:49.266653Z 2021-12-17T14:19:49.222429Z 2021-12-17T14:19:49.172079Z 2021-12-17T14:19:41.262162Z 2021-12-17T14:19:40.965569Z 2021-12-17T14:19:40.606065Z 2021-12-17T14:19:40.408366Z 2021-12-17T14:19:40.033429Z 2021-12-17T14:19:39.654251Z 2021-12-17T14:19:39.274373Z 2021-12-17T14:19:38.949219Z 2021-12-17T14:19:36.264067Z 2021-12-17T14:19:36.112082Z 2021-12-17T14:19:35.104704Z", + "modified_diff": 43.5488778, + "started": "2021-12-17T14:00:05.394270Z 2021-12-17T14:00:06.601247Z 2021-12-17T14:00:04.914123Z 2021-12-17T14:00:05.922374Z 2021-12-17T14:00:04.529206Z 2021-12-17T14:00:05.508756Z 2021-12-17T14:00:04.275263Z 2021-12-17T14:00:02.460540Z 2021-12-17T14:00:01.122044Z 2021-12-17T14:00:01.544915Z 2021-12-17T14:00:00.763381Z 2021-12-17T14:00:00.588340Z 2021-12-17T13:59:59.875562Z 2021-12-17T13:59:59.948384Z 2021-12-17T13:59:59.369771Z 2021-12-17T13:59:58.838895Z 2021-12-17T13:59:58.601746Z 2021-12-17T13:59:59.082199Z 2021-12-17T13:59:58.096256Z 2021-12-17T13:59:57.726108Z 2021-12-17T13:59:57.451934Z 2021-12-17T13:59:57.782012Z 2021-12-17T13:59:56.943835Z 2021-12-17T13:59:56.660653Z 2021-12-17T13:59:56.502843Z 2021-12-17T13:59:56.307749Z 2021-12-17T13:59:55.729964Z 2021-12-17T13:59:55.651230Z 2021-12-17T13:59:52.303110Z 2021-12-17T13:59:52.030971Z 2021-12-17T13:59:51.693769Z 2021-12-17T13:59:51.485689Z 2021-12-17T13:59:51.199548Z 2021-12-17T13:59:49.718017Z 2021-12-17T13:59:49.546845Z 2021-12-17T13:59:48.028495Z 2021-12-17T14:00:34.397132Z 2021-12-17T14:00:33.796971Z 2021-12-17T14:00:33.610198Z 2021-12-17T14:00:32.715766Z 2021-12-17T14:00:27.603378Z 2021-12-17T14:00:26.647201Z 2021-12-17T14:00:26.318346Z 2021-12-17T14:00:25.969991Z 2021-12-17T14:00:26.680690Z 2021-12-17T14:00:25.514624Z 2021-12-17T14:00:25.860210Z 2021-12-17T14:00:25.038352Z 2021-12-17T14:00:26.307159Z 2021-12-17T14:00:24.637924Z 2021-12-17T14:00:26.332813Z 2021-12-17T14:00:24.161049Z 2021-12-17T14:00:25.203421Z 2021-12-17T14:00:23.865013Z 2021-12-17T14:00:25.540639Z 2021-12-17T14:00:23.901193Z 2021-12-17T14:00:19.726450Z 2021-12-17T14:00:18.382957Z 2021-12-17T14:00:20.857263Z 2021-12-17T14:00:23.812715Z 2021-12-17T14:00:17.692458Z 2021-12-17T14:00:18.537925Z 2021-12-17T14:00:17.004495Z 2021-12-17T14:00:17.544598Z 2021-12-17T14:00:16.181180Z 2021-12-17T14:00:16.677817Z 2021-12-17T14:00:15.504796Z 2021-12-17T14:00:16.489503Z 2021-12-17T14:00:15.076081Z 2021-12-17T14:00:15.206749Z 2021-12-17T14:00:14.563087Z 2021-12-17T14:00:14.518024Z 2021-12-17T14:00:14.046076Z 2021-12-17T14:00:13.897643Z 2021-12-17T14:00:13.167256Z 2021-12-17T14:00:12.704182Z 2021-12-17T14:00:12.140020Z 2021-12-17T14:00:13.307999Z 2021-12-17T14:00:11.483674Z 2021-12-17T14:00:12.292464Z 2021-12-17T14:00:10.863134Z 2021-12-17T14:00:11.639138Z 2021-12-17T14:00:10.342931Z 2021-12-17T14:00:11.399391Z 2021-12-17T14:00:09.733983Z 2021-12-17T14:00:10.825520Z 2021-12-17T14:00:09.087888Z 2021-12-17T14:00:10.150584Z 2021-12-17T14:00:08.472932Z 2021-12-17T14:00:09.989392Z 2021-12-17T14:00:08.019907Z 2021-12-17T14:00:09.871115Z 2021-12-17T14:00:07.354881Z 2021-12-17T14:00:07.784247Z 2021-12-17T14:00:06.840330Z 2021-12-17T14:00:08.166232Z 2021-12-17T14:00:06.381893Z 2021-12-17T14:00:07.960745Z 2021-12-17T14:00:05.808799Z 2021-12-17T14:00:07.085271Z 2021-12-17T14:06:19.588090Z 2021-12-17T14:06:18.873446Z 2021-12-17T14:06:18.635756Z 2021-12-17T14:06:17.135733Z 2021-12-17T14:06:16.607434Z 2021-12-17T14:06:16.121367Z 2021-12-17T14:06:16.092017Z 2021-12-17T14:06:15.082098Z 2021-12-17T14:06:15.100397Z 2021-12-17T14:06:13.968819Z 2021-12-17T14:06:13.749244Z 2021-12-17T14:06:13.028923Z 2021-12-17T14:06:12.383377Z 2021-12-17T14:06:11.932272Z 2021-12-17T14:06:11.518648Z 2021-12-17T14:06:10.934808Z 2021-12-17T14:06:10.728639Z 2021-12-17T14:06:09.457094Z 2021-12-17T14:06:09.797098Z 2021-12-17T14:06:08.712850Z 2021-12-17T14:06:08.544548Z 2021-12-17T14:06:07.803129Z 2021-12-17T14:06:07.837558Z 2021-12-17T14:06:06.957531Z 2021-12-17T14:06:06.558179Z 2021-12-17T14:06:05.718162Z 2021-12-17T14:06:05.239033Z 2021-12-17T14:06:04.612046Z 2021-12-17T14:06:03.644044Z 2021-12-17T14:06:03.461607Z 2021-12-17T14:06:03.481341Z 2021-12-17T14:06:02.343176Z 2021-12-17T14:06:03.069362Z 2021-12-17T14:06:01.772278Z 2021-12-17T14:06:01.518113Z 2021-12-17T14:06:01.119818Z 2021-12-17T14:06:00.817225Z 2021-12-17T14:05:59.659523Z 2021-12-17T14:05:59.106320Z 2021-12-17T14:05:58.372805Z 2021-12-17T14:05:58.187289Z 2021-12-17T14:05:57.475694Z 2021-12-17T14:05:57.136693Z 2021-12-17T14:05:56.208284Z 2021-12-17T14:05:55.624319Z 2021-12-17T14:05:54.886846Z 2021-12-17T14:05:55.402217Z 2021-12-17T14:05:54.434176Z 2021-12-17T14:05:54.059866Z 2021-12-17T14:05:53.159899Z 2021-12-17T14:05:53.231976Z 2021-12-17T14:05:52.512761Z 2021-12-17T14:05:52.490155Z 2021-12-17T14:05:51.623024Z 2021-12-17T14:05:51.293201Z 2021-12-17T14:05:51.235515Z 2021-12-17T14:05:50.359680Z 2021-12-17T14:05:49.583223Z 2021-12-17T14:05:49.697015Z 2021-12-17T14:05:49.287329Z 2021-12-17T14:05:49.014850Z 2021-12-17T14:05:48.604493Z 2021-12-17T14:05:48.358172Z 2021-12-17T14:05:48.097114Z 2021-12-17T14:05:47.795217Z 2021-12-17T14:05:47.372825Z 2021-12-17T14:05:47.325014Z 2021-12-17T14:05:47.145368Z 2021-12-17T14:05:36.219154Z 2021-12-17T14:05:36.202445Z 2021-12-17T14:05:35.684029Z 2021-12-17T14:05:35.290157Z 2021-12-17T14:05:34.551964Z 2021-12-17T14:05:34.300341Z 2021-12-17T14:05:33.682761Z 2021-12-17T14:05:33.361116Z 2021-12-17T14:05:32.784922Z 2021-12-17T14:05:32.694143Z 2021-12-17T14:05:31.895758Z 2021-12-17T14:05:31.510528Z 2021-12-17T14:05:30.833728Z 2021-12-17T14:05:31.518042Z 2021-12-17T14:05:30.195091Z 2021-12-17T14:05:31.027448Z 2021-12-17T14:05:28.998140Z 2021-12-17T14:05:28.768460Z 2021-12-17T14:05:28.391926Z 2021-12-17T14:05:27.943015Z 2021-12-17T14:05:27.475635Z 2021-12-17T14:05:27.285493Z 2021-12-17T14:05:13.890786Z 2021-12-17T14:05:13.990205Z 2021-12-17T14:05:13.596836Z 2021-12-17T14:05:13.736186Z 2021-12-17T14:05:13.052561Z 2021-12-17T14:05:12.889767Z 2021-12-17T14:05:12.414590Z 2021-12-17T14:05:11.089108Z 2021-12-17T14:05:10.837730Z 2021-12-17T14:05:09.378663Z 2021-12-17T14:10:55.837946Z 2021-12-17T14:10:55.437401Z 2021-12-17T14:10:54.648970Z 2021-12-17T14:10:53.989807Z 2021-12-17T14:10:53.936503Z 2021-12-17T14:10:52.931634Z 2021-12-17T14:10:51.548661Z 2021-12-17T14:10:51.152578Z 2021-12-17T14:10:50.493083Z 2021-12-17T14:10:50.356849Z 2021-12-17T14:10:49.394782Z 2021-12-17T14:10:49.128294Z 2021-12-17T14:10:48.552758Z 2021-12-17T14:10:48.289063Z 2021-12-17T14:10:48.080028Z 2021-12-17T14:10:47.865107Z 2021-12-17T14:10:47.402359Z 2021-12-17T14:10:47.153226Z 2021-12-17T14:10:46.155710Z 2021-12-17T14:10:45.863015Z 2021-12-17T14:10:45.210998Z 2021-12-17T14:10:44.881621Z 2021-12-17T14:10:44.298843Z 2021-12-17T14:10:44.137402Z 2021-12-17T14:10:43.479851Z 2021-12-17T14:10:43.058179Z 2021-12-17T14:10:42.363054Z 2021-12-17T14:10:42.028574Z 2021-12-17T14:10:41.415039Z 2021-12-17T14:10:41.149634Z 2021-12-17T14:10:40.571372Z 2021-12-17T14:10:40.230114Z 2021-12-17T14:10:39.582219Z 2021-12-17T14:10:39.182260Z 2021-12-17T14:10:38.964082Z 2021-12-17T14:10:38.630421Z 2021-12-17T14:10:37.866164Z 2021-12-17T14:10:37.522580Z 2021-12-17T14:10:37.005173Z 2021-12-17T14:10:36.586482Z 2021-12-17T14:10:35.767972Z 2021-12-17T14:10:35.844815Z 2021-12-17T14:10:35.224147Z 2021-12-17T14:10:35.002050Z 2021-12-17T14:10:34.559969Z 2021-12-17T14:10:34.420333Z 2021-12-17T14:10:34.080687Z 2021-12-17T14:10:34.075615Z 2021-12-17T14:10:33.548974Z 2021-12-17T14:10:33.383097Z 2021-12-17T14:10:33.120035Z 2021-12-17T14:10:25.243624Z 2021-12-17T14:10:24.590178Z 2021-12-17T14:10:33.097216Z 2021-12-17T14:10:24.302963Z 2021-12-17T14:10:23.598131Z 2021-12-17T14:10:23.206257Z 2021-12-17T14:10:22.804409Z 2021-12-17T14:10:22.636101Z 2021-12-17T14:10:21.969781Z 2021-12-17T14:10:21.288296Z 2021-12-17T14:10:21.143192Z 2021-12-17T14:10:20.908446Z 2021-12-17T14:10:20.283331Z 2021-12-17T14:10:19.905906Z 2021-12-17T14:10:19.959819Z 2021-12-17T14:10:19.442270Z 2021-12-17T14:10:19.591862Z 2021-12-17T14:10:18.633127Z 2021-12-17T14:10:18.176621Z 2021-12-17T14:10:17.412639Z 2021-12-17T14:10:17.943811Z 2021-12-17T14:10:17.346172Z 2021-12-17T14:10:16.777016Z 2021-12-17T14:10:16.109601Z 2021-12-17T14:10:15.539323Z 2021-12-17T14:10:15.152447Z 2021-12-17T14:10:14.896390Z 2021-12-17T14:10:14.743736Z 2021-12-17T14:10:14.072275Z 2021-12-17T14:10:13.913054Z 2021-12-17T14:10:13.507925Z 2021-12-17T14:10:13.436141Z 2021-12-17T14:10:13.220268Z 2021-12-17T14:10:12.888136Z 2021-12-17T14:10:12.578195Z 2021-12-17T14:10:12.406026Z 2021-12-17T14:10:12.249279Z 2021-12-17T14:10:12.063909Z 2021-12-17T14:10:11.909765Z 2021-12-17T14:10:01.067733Z 2021-12-17T14:10:00.119371Z 2021-12-17T14:09:59.508028Z 2021-12-17T14:09:59.292321Z 2021-12-17T14:09:58.290310Z 2021-12-17T14:09:57.882196Z 2021-12-17T14:09:54.849053Z 2021-12-17T14:09:53.874105Z 2021-12-17T14:09:53.768655Z 2021-12-17T14:09:53.128159Z 2021-12-17T14:15:45.557752Z 2021-12-17T14:15:46.151336Z 2021-12-17T14:15:45.004324Z 2021-12-17T14:15:45.142704Z 2021-12-17T14:15:44.705532Z 2021-12-17T14:15:44.094455Z 2021-12-17T14:15:44.074972Z 2021-12-17T14:15:42.902194Z 2021-12-17T14:15:42.597936Z 2021-12-17T14:15:41.779458Z 2021-12-17T14:15:41.678040Z 2021-12-17T14:15:40.957768Z 2021-12-17T14:15:39.814659Z 2021-12-17T14:15:39.767737Z 2021-12-17T14:15:39.318949Z 2021-12-17T14:15:38.815479Z 2021-12-17T14:15:38.393567Z 2021-12-17T14:15:38.432361Z 2021-12-17T14:15:37.996083Z 2021-12-17T14:15:37.104796Z 2021-12-17T14:15:38.124040Z 2021-12-17T14:15:36.869032Z 2021-12-17T14:15:37.926720Z 2021-12-17T14:15:36.689514Z 2021-12-17T14:15:34.980137Z 2021-12-17T14:15:34.183364Z 2021-12-17T14:15:33.264003Z 2021-12-17T14:15:32.910687Z 2021-12-17T14:15:31.884631Z 2021-12-17T14:15:31.547631Z 2021-12-17T14:15:30.916785Z 2021-12-17T14:15:30.395314Z 2021-12-17T14:15:30.226828Z 2021-12-17T14:15:29.539375Z 2021-12-17T14:15:29.148114Z 2021-12-17T14:15:28.768067Z 2021-12-17T14:15:28.398044Z 2021-12-17T14:15:27.738238Z 2021-12-17T14:15:27.172960Z 2021-12-17T14:15:26.990408Z 2021-12-17T14:15:26.619984Z 2021-12-17T14:15:26.354349Z 2021-12-17T14:15:25.691305Z 2021-12-17T14:15:25.323493Z 2021-12-17T14:15:24.896026Z 2021-12-17T14:15:24.395226Z 2021-12-17T14:15:24.084001Z 2021-12-17T14:15:23.883184Z 2021-12-17T14:15:23.274485Z 2021-12-17T14:15:22.963011Z 2021-12-17T14:15:22.576377Z 2021-12-17T14:15:22.430159Z 2021-12-17T14:15:21.889170Z 2021-12-17T14:15:21.188261Z 2021-12-17T14:15:20.865936Z 2021-12-17T14:15:20.439563Z 2021-12-17T14:15:20.080813Z 2021-12-17T14:15:19.694234Z 2021-12-17T14:15:19.139538Z 2021-12-17T14:15:18.673010Z 2021-12-17T14:15:18.444908Z 2021-12-17T14:15:18.053186Z 2021-12-17T14:15:17.823128Z 2021-12-17T14:15:17.537079Z 2021-12-17T14:15:17.387063Z 2021-12-17T14:15:07.502258Z 2021-12-17T14:15:07.102244Z 2021-12-17T14:15:06.163896Z 2021-12-17T14:15:06.394089Z 2021-12-17T14:15:05.483190Z 2021-12-17T14:15:05.569632Z 2021-12-17T14:15:05.040166Z 2021-12-17T14:15:04.651248Z 2021-12-17T14:15:04.375315Z 2021-12-17T14:15:04.022435Z 2021-12-17T14:15:03.754291Z 2021-12-17T14:15:03.492088Z 2021-12-17T14:15:03.234069Z 2021-12-17T14:15:02.553575Z 2021-12-17T14:15:02.196366Z 2021-12-17T14:15:02.172085Z 2021-12-17T14:15:01.774704Z 2021-12-17T14:15:01.373084Z 2021-12-17T14:15:00.913111Z 2021-12-17T14:15:00.740041Z 2021-12-17T14:15:00.108577Z 2021-12-17T14:14:59.944266Z 2021-12-17T14:14:59.743675Z 2021-12-17T14:14:59.710906Z 2021-12-17T14:14:59.423159Z 2021-12-17T14:14:59.313481Z 2021-12-17T14:14:59.054583Z 2021-12-17T14:14:58.873589Z 2021-12-17T14:14:58.948396Z 2021-12-17T14:14:58.535162Z 2021-12-17T14:14:58.481070Z 2021-12-17T14:14:50.167774Z 2021-12-17T14:14:49.796446Z 2021-12-17T14:14:49.364145Z 2021-12-17T14:14:46.492371Z 2021-12-17T14:20:47.465644Z 2021-12-17T14:20:46.449968Z 2021-12-17T14:20:46.301969Z 2021-12-17T14:20:45.504727Z 2021-12-17T14:20:44.604336Z 2021-12-17T14:20:43.971169Z 2021-12-17T14:20:43.989722Z 2021-12-17T14:20:43.351192Z 2021-12-17T14:20:43.100500Z 2021-12-17T14:20:42.940030Z 2021-12-17T14:20:42.911866Z 2021-12-17T14:20:22.396328Z 2021-12-17T14:20:42.295180Z 2021-12-17T14:20:22.337853Z 2021-12-17T14:20:21.503326Z 2021-12-17T14:20:21.402800Z 2021-12-17T14:20:21.404261Z 2021-12-17T14:20:20.306376Z 2021-12-17T14:20:20.241675Z 2021-12-17T14:20:42.262169Z 2021-12-17T14:20:19.298728Z 2021-12-17T14:20:19.527430Z 2021-12-17T14:20:18.552466Z 2021-12-17T14:20:18.517026Z 2021-12-17T14:20:18.160147Z 2021-12-17T14:20:16.905339Z 2021-12-17T14:20:16.334850Z 2021-12-17T14:20:16.108322Z 2021-12-17T14:20:15.154162Z 2021-12-17T14:20:15.597117Z 2021-12-17T14:20:15.017713Z 2021-12-17T14:20:15.234590Z 2021-12-17T14:20:41.915899Z 2021-12-17T14:20:14.123852Z 2021-12-17T14:20:14.995223Z 2021-12-17T14:20:13.850762Z 2021-12-17T14:20:12.914279Z 2021-12-17T14:20:12.752458Z 2021-12-17T14:20:12.811574Z 2021-12-17T14:20:41.651012Z 2021-12-17T14:20:11.882970Z 2021-12-17T14:20:12.579210Z 2021-12-17T14:20:11.174443Z 2021-12-17T14:20:10.404834Z 2021-12-17T14:20:10.613017Z 2021-12-17T14:20:09.949285Z 2021-12-17T14:20:09.346168Z 2021-12-17T14:20:08.814682Z 2021-12-17T14:20:08.557093Z 2021-12-17T14:20:08.004574Z 2021-12-17T14:20:07.361006Z 2021-12-17T14:20:07.416400Z 2021-12-17T14:20:06.337117Z 2021-12-17T14:20:06.504316Z 2021-12-17T14:20:05.638983Z 2021-12-17T14:20:05.741624Z 2021-12-17T14:20:05.030177Z 2021-12-17T14:20:04.931822Z 2021-12-17T14:20:03.952382Z 2021-12-17T14:20:03.769419Z 2021-12-17T14:20:03.538287Z 2021-12-17T14:20:03.040523Z 2021-12-17T14:20:02.669460Z 2021-12-17T14:20:02.821701Z 2021-12-17T14:20:01.968986Z 2021-12-17T14:20:01.871841Z 2021-12-17T14:20:01.107163Z 2021-12-17T14:20:01.268955Z 2021-12-17T14:20:00.864186Z 2021-12-17T14:20:00.659753Z 2021-12-17T14:20:00.290388Z 2021-12-17T14:19:59.372718Z 2021-12-17T14:19:59.504112Z 2021-12-17T14:19:58.863384Z 2021-12-17T14:19:58.534828Z 2021-12-17T14:19:58.045147Z 2021-12-17T14:19:56.851163Z 2021-12-17T14:19:56.463926Z 2021-12-17T14:19:55.981516Z 2021-12-17T14:19:55.867091Z 2021-12-17T14:19:55.560578Z 2021-12-17T14:19:55.137429Z 2021-12-17T14:19:54.862287Z 2021-12-17T14:19:54.699221Z 2021-12-17T14:19:54.393173Z 2021-12-17T14:19:54.199143Z 2021-12-17T14:19:53.963835Z 2021-12-17T14:19:53.670190Z 2021-12-17T14:19:53.566854Z 2021-12-17T14:19:46.581899Z 2021-12-17T14:19:46.531724Z 2021-12-17T14:19:45.505017Z 2021-12-17T14:19:45.288050Z 2021-12-17T14:19:44.473682Z 2021-12-17T14:19:42.864264Z 2021-12-17T14:19:43.582399Z 2021-12-17T14:19:42.273809Z 2021-12-17T14:19:37.230058Z 2021-12-17T14:19:36.820749Z 2021-12-17T14:19:35.415317Z", + "started_diff": 62.1994286, + "successful_job_ids": " [99, 98, 97, 95, 94, 92, 91, 90, 89, 88, 87, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 173, 172, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 155, 154, 153, 152, 151, 150, 148, 146, 145, 144, 143, 141, 140, 138, 137, 135, 134, 133, 132, 130, 129, 128, 127, 125, 124, 123, 122, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 273, 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 473, 472, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 573, 572, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474]" + }, + "started": "2021-12-17T14:48:48.214269+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-api.json new file mode 100644 index 00000000..8d318d75 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-19T01:45:41.897946+00:00", + "golden": "true", + "id": "run-2021-12-18T20_12_01_863_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2450.7092560495958, + "max": 26166.6, + "mean": 5279.0706441980365, + "median": 3327.9666666666667, + "min": 1427.2, + "non_zero_mean": 5279.0706441980365, + "non_zero_median": 3327.9666666666667, + "percentile25": 2718.90741061707, + "percentile75": 5169.616666666666, + "percentile90": 12460.806666666673, + "percentile99": 22792.16533333334, + "percentile999": 25427.262133333432, + "range": 24739.399999999998, + "samples": 308, + "stdev": 4600.253489766332, + "sum": 1625953.758412995 + } + }, + "cpu": { + "system": { + "iqr": 0.27016666666668054, + "max": 2.4433333333333165, + "mean": 0.3616104047776143, + "median": 0.15300000000000105, + "min": 0.045333333333333795, + "non_zero_mean": 0.3616104047776143, + "non_zero_median": 0.15300000000000105, + "percentile25": 0.07866666666665954, + "percentile75": 0.3488333333333401, + "percentile90": 1.2686000000000015, + "percentile99": 2.152226666666653, + "percentile999": 2.4105866666666618, + "range": 2.397999999999983, + "samples": 308, + "stdev": 0.48987712275113043, + "sum": 111.37600467150529 + }, + "user": { + "iqr": 0.7065606373757961, + "max": 5.967333333333379, + "mean": 1.3830670955739492, + "median": 0.827333333333316, + "min": 0.4039999999999964, + "non_zero_mean": 1.3830670955739492, + "non_zero_median": 0.827333333333316, + "percentile25": 0.6246666666666595, + "percentile75": 1.3312273040424556, + "percentile90": 3.908466666666673, + "percentile99": 5.714893333333297, + "percentile999": 5.913301333333374, + "range": 5.563333333333382, + "samples": 308, + "stdev": 1.340891354947463, + "sum": 425.9846654367764 + } + }, + "entropy_available_bits": { + "iqr": 1.9999999999999998, + "max": 214.79999999999998, + "mean": 20.93160144324194, + "median": 1.7333333333333334, + "min": 0.3333333333333333, + "non_zero_mean": 20.93160144324194, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.8, + "percentile75": 2.8, + "percentile90": 130.06000000000003, + "percentile99": 213.12866666666665, + "percentile999": 214.37020000000004, + "range": 214.46666666666664, + "samples": 308, + "stdev": 57.23313811547121, + "sum": 6446.933244518525 + }, + "memory": { + "used": { + "iqr": 383664128.0, + "max": 16143212544.0, + "mean": 6485245021.090909, + "median": 5998968832.0, + "min": 5348274176.0, + "non_zero_mean": 6485245021.090909, + "non_zero_median": 5998968832.0, + "percentile25": 5880256512.0, + "percentile75": 6263920640.0, + "percentile90": 7147610112.000001, + "percentile99": 13619717201.920002, + "percentile999": 15905429618.688032, + "range": 10794938368.0, + "samples": 308, + "stdev": 1579667590.8837585, + "sum": 1997455466496.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 31.266666666666666, + "mean": 0.24264069264069263, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9.341666666666667, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.7333333333333334, + "percentile999": 29.17906666666694, + "range": 31.266666666666666, + "samples": 308, + "stdev": 2.365217236736651, + "sum": 74.73333333333335 + } + }, + "writes_completed": { + "total": { + "iqr": 87.25, + "max": 1905.5333333333335, + "mean": 121.69741212362679, + "median": 31.866666666666667, + "min": 0.0, + "non_zero_mean": 123.29869386209556, + "non_zero_median": 33.10000000000001, + "percentile25": 1.5999999999999999, + "percentile75": 88.85, + "percentile90": 395.88000000000005, + "percentile99": 1235.682666666667, + "percentile999": 1740.7152666666886, + "range": 1905.5333333333335, + "samples": 308, + "stdev": 254.98550802658963, + "sum": 37482.802934077066 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 913.8666666666668, + "max": 3274.9333333333334, + "mean": 1537.478354978355, + "median": 1410.8333333333335, + "min": 467.0, + "non_zero_mean": 1537.478354978355, + "non_zero_median": 1410.8333333333335, + "percentile25": 1102.5666666666666, + "percentile75": 2016.4333333333334, + "percentile90": 2252.84, + "percentile99": 2782.486666666667, + "percentile999": 3134.6957333333517, + "range": 2807.9333333333334, + "samples": 308, + "stdev": 551.9382202302049, + "sum": 473543.33333333355 + } + }, + "cpu": { + "system": { + "iqr": 0.052333333333331206, + "max": 0.29466666666666014, + "mean": 0.10044588744588744, + "median": 0.08533333333333151, + "min": 0.019333333333330906, + "non_zero_mean": 0.10044588744588744, + "non_zero_median": 0.08533333333333151, + "percentile25": 0.0660000000000025, + "percentile75": 0.1183333333333337, + "percentile90": 0.1838666666666685, + "percentile99": 0.2541533333333364, + "percentile999": 0.29200599999999627, + "range": 0.2753333333333292, + "samples": 308, + "stdev": 0.052849244695637385, + "sum": 30.937333333333328 + }, + "user": { + "iqr": 0.1243333333333349, + "max": 1.0640000000000025, + "mean": 0.20282900432900433, + "median": 0.20033333333333303, + "min": 0.021333333333329086, + "non_zero_mean": 0.20282900432900433, + "non_zero_median": 0.20033333333333303, + "percentile25": 0.13383333333333097, + "percentile75": 0.2581666666666659, + "percentile90": 0.3194000000000025, + "percentile99": 0.5684533333333327, + "percentile999": 0.9190960000000219, + "range": 1.0426666666666735, + "samples": 308, + "stdev": 0.10960394423618534, + "sum": 62.471333333333334 + } + }, + "entropy_available_bits": { + "iqr": 1.6666666666666667, + "max": 210.93333333333334, + "mean": 11.792857142857143, + "median": 0.8666666666666667, + "min": 0.0, + "non_zero_mean": 20.17888888888889, + "non_zero_median": 1.4, + "percentile25": 0.0, + "percentile75": 1.6666666666666667, + "percentile90": 3.020000000000001, + "percentile99": 210.86666666666667, + "percentile999": 210.93333333333334, + "range": 210.93333333333334, + "samples": 308, + "stdev": 46.65857281513147, + "sum": 3632.1999999999994 + }, + "memory": { + "used": { + "iqr": 24003584.0, + "max": 1168117760.0, + "mean": 444723479.27272725, + "median": 391542784.0, + "min": 373960704.0, + "non_zero_mean": 444723479.27272725, + "non_zero_median": 391542784.0, + "percentile25": 384544768.0, + "percentile75": 408548352.0, + "percentile90": 555960729.6, + "percentile99": 1094156738.5600002, + "percentile999": 1149512204.2880025, + "range": 794157056.0, + "samples": 308, + "stdev": 150856390.94422263, + "sum": 136974831616.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 5461.333333333333, + "max": 59660756.2, + "mean": 1151482.4922077921, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3253730.3449541284, + "non_zero_median": 10922.666666666666, + "percentile25": 0.0, + "percentile75": 5461.333333333333, + "percentile90": 30747.306666666675, + "percentile99": 57858619.965333335, + "percentile999": 59586816.846400015, + "range": 59660756.2, + "samples": 308, + "stdev": 8113355.000127413, + "sum": 354656607.5999999 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 455.587137, + "mean": 3.9417839025974026, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 31.949195842105265, + "non_zero_median": 0.056656, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0006485000000000004, + "percentile99": 167.0411926000012, + "percentile999": 404.9337155940067, + "range": 455.587137, + "samples": 308, + "stdev": 35.885054699844225, + "sum": 1214.069442 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.451644, + "max": 470.580223, + "mean": 25.378795204545455, + "median": 0.0, + "min": -0.000544, + "non_zero_mean": 55.83334945, + "non_zero_median": 1.1614865, + "percentile25": 0.0, + "percentile75": 0.451644, + "percentile90": 68.1837640000001, + "percentile99": 409.5285401700001, + "percentile999": 461.37293503800123, + "range": 470.580767, + "samples": 308, + "stdev": 80.25364608122675, + "sum": 7816.668923000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 5.866666666666666, + "mean": 0.18246753246753247, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.7466666666666666, + "non_zero_median": 3.466666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 5.105333333333337, + "percentile999": 5.743866666666682, + "range": 5.866666666666666, + "samples": 308, + "stdev": 0.8546117091349085, + "sum": 56.20000000000001 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 10000.866666666667, + "mean": 399.90497835497837, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8211.382222222222, + "non_zero_median": 8186.866666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9962.046000000002, + "percentile999": 9999.147466666667, + "range": 10000.866666666667, + "samples": 308, + "stdev": 1807.5393455290675, + "sum": 123170.73333333334 + }, + "pg_stat_database_blks_hit": { + "iqr": 21304.966666666667, + "max": 204060.06666666668, + "mean": 22599.051948051947, + "median": 22042.133333333335, + "min": 2390.9333333333334, + "non_zero_mean": 22599.051948051947, + "non_zero_median": 22042.133333333335, + "percentile25": 8958.933333333334, + "percentile75": 30263.9, + "percentile90": 39606.77333333335, + "percentile99": 50168.112666666675, + "percentile999": 190729.79920000175, + "range": 201669.13333333336, + "samples": 308, + "stdev": 18524.080214499845, + "sum": 6960508.000000008 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5333333333333333, + "max": 215.4, + "mean": 1.2344155844155844, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.590794979079498, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.6866666666666674, + "percentile99": 3.8526666666666682, + "percentile999": 151.3802666666751, + "range": 215.4, + "samples": 308, + "stdev": 12.273219063147366, + "sum": 380.1999999999995 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 308, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5333333333333333, + "max": 215.4, + "mean": 1.2344155844155844, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.590794979079498, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6, + "percentile90": 1.6866666666666674, + "percentile99": 3.8526666666666682, + "percentile999": 151.3802666666751, + "range": 215.4, + "samples": 308, + "stdev": 12.273219063147366, + "sum": 380.1999999999995 + }, + "pg_stat_database_tup_deleted": { + "iqr": 1.9833333333333334, + "max": 4005.866666666667, + "mean": 35.254978354978356, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 92.02146892655367, + "non_zero_median": 11.533333333333335, + "percentile25": 0.0, + "percentile75": 1.9833333333333334, + "percentile90": 55.6, + "percentile99": 72.37733333333337, + "percentile999": 4004.5977333333335, + "range": 4005.866666666667, + "samples": 308, + "stdev": 321.98661087506116, + "sum": 10858.533333333335 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11367.216666666667, + "max": 359497.8, + "mean": 12803.053463203463, + "median": 12287.0, + "min": 1325.8666666666666, + "non_zero_mean": 12803.053463203463, + "non_zero_median": 12287.0, + "percentile25": 4277.783333333333, + "percentile75": 15645.0, + "percentile90": 22147.90666666667, + "percentile99": 31635.42866666667, + "percentile999": 261293.20733334628, + "range": 358171.93333333335, + "samples": 308, + "stdev": 21243.51743815859, + "sum": 3943340.46666667 + }, + "pg_stat_database_tup_inserted": { + "iqr": 7.866666666666667, + "max": 8005.333333333333, + "mean": 39.57012987012987, + "median": 2.1, + "min": 0.0, + "non_zero_mean": 46.51755725190839, + "non_zero_median": 2.933333333333333, + "percentile25": 1.0, + "percentile75": 8.866666666666667, + "percentile90": 52.52, + "percentile99": 129.1686666666667, + "percentile999": 5593.132000000318, + "range": 8005.333333333333, + "samples": 308, + "stdev": 456.30394861756207, + "sum": 12187.599999999991 + }, + "pg_stat_database_tup_returned": { + "iqr": 31424.800000000003, + "max": 525938.0, + "mean": 39078.61688311688, + "median": 30594.9, + "min": 3286.3333333333335, + "non_zero_mean": 39078.61688311688, + "non_zero_median": 30594.9, + "percentile25": 14641.616666666667, + "percentile75": 46066.41666666667, + "percentile90": 81243.15333333335, + "percentile99": 146984.4513333334, + "percentile999": 490959.9140666713, + "range": 522651.6666666667, + "samples": 308, + "stdev": 45812.62994119375, + "sum": 12036213.999999985 + }, + "pg_stat_database_tup_updated": { + "iqr": 11.533333333333335, + "max": 62.86666666666667, + "mean": 9.013636363636364, + "median": 5.9, + "min": 0.13333333333333333, + "non_zero_mean": 9.013636363636364, + "non_zero_median": 5.9, + "percentile25": 2.8166666666666664, + "percentile75": 14.350000000000001, + "percentile90": 21.419999999999998, + "percentile99": 43.28066666666671, + "percentile999": 57.72953333333401, + "range": 62.733333333333334, + "samples": 308, + "stdev": 8.855180296105257, + "sum": 2776.2000000000007 + }, + "pg_stat_database_xact_commit": { + "iqr": 113.44999999999999, + "max": 649.7333333333333, + "mean": 111.51601731601731, + "median": 78.96666666666667, + "min": 5.866666666666666, + "non_zero_mean": 111.51601731601731, + "non_zero_median": 78.96666666666667, + "percentile25": 35.833333333333336, + "percentile75": 149.28333333333333, + "percentile90": 249.22000000000003, + "percentile99": 522.34, + "percentile999": 645.9879333333338, + "range": 643.8666666666667, + "samples": 308, + "stdev": 113.26789726320293, + "sum": 34346.93333333333 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.11774891774891774, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.11774891774891774, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4, + "percentile99": 0.5333333333333333, + "percentile999": 0.579533333333336, + "range": 0.5333333333333333, + "samples": 308, + "stdev": 0.13054460557660502, + "sum": 36.26666666666682 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 308, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 25.566666666666674, + "max": 169.2, + "mean": 36.1030303030303, + "median": 30.333333333333332, + "min": 1.3333333333333333, + "non_zero_mean": 36.1030303030303, + "non_zero_median": 30.333333333333332, + "percentile25": 21.333333333333332, + "percentile75": 46.900000000000006, + "percentile90": 65.75333333333333, + "percentile99": 109.04866666666673, + "percentile999": 156.79720000000162, + "range": 167.86666666666665, + "samples": 308, + "stdev": 22.022731113197644, + "sum": 11119.733333333332 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-19T00:28:51.615430+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-18T20_12_01_863_0000", + "ended": "2021-12-19T01:45:36.000072+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.032271416498588223, + "iterations": 1, + "max": 0.6326546480013349, + "mean": 0.6018956070000665, + "median": 0.5999106319995917, + "min": 0.5657633139999234, + "rounds": 15, + "stddev": 0.019162778077146007 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.012634885749776004, + "iterations": 1, + "max": 0.6196494870000606, + "mean": 0.6003435528666159, + "median": 0.6040499829996406, + "min": 0.5711472450002475, + "rounds": 15, + "stddev": 0.012693044112406392 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.021557146001214278, + "iterations": 1, + "max": 0.47727292499985197, + "mean": 0.4549285254004644, + "median": 0.45383848800065607, + "min": 0.4314854029998969, + "rounds": 10, + "stddev": 0.014204366830352252 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.010460638500262576, + "iterations": 1, + "max": 0.33377842499976396, + "mean": 0.2974243550999745, + "median": 0.29629654650034354, + "min": 0.2820515790008358, + "rounds": 100, + "stddev": 0.008265381624039177 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.02091518599900155, + "iterations": 1, + "max": 0.7655058609998378, + "mean": 0.541305114190127, + "median": 0.5369744745003118, + "min": 0.5080825049990381, + "rounds": 100, + "stddev": 0.03056040521158747 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.15824251599951822, + "iterations": 1, + "max": 1.6990842359991802, + "mean": 1.5273062242999003, + "median": 1.4766304044997014, + "min": 1.4501396289997501, + "rounds": 10, + "stddev": 0.09272658034987952 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0015693864997956553, + "iterations": 1, + "max": 0.09432522900169715, + "mean": 0.08189868650045658, + "median": 0.07981010650109965, + "min": 0.0778963400007342, + "rounds": 12, + "stddev": 0.00526641350955857 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.010926617250788695, + "iterations": 1, + "max": 0.10014883799885865, + "mean": 0.08791308390887438, + "median": 0.08986428599928331, + "min": 0.07977232799930789, + "rounds": 11, + "stddev": 0.0071866024024107335 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.026104660000783042, + "iterations": 1, + "max": 0.13933023400022648, + "mean": 0.12050184340023407, + "median": 0.1204099500000666, + "min": 0.10407308200046828, + "rounds": 10, + "stddev": 0.013703990978128631 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.007407971999782603, + "iterations": 1, + "max": 0.15144359999976587, + "mean": 0.12724179800006824, + "median": 0.12188715850061271, + "min": 0.11485782999989169, + "rounds": 10, + "stddev": 0.012713378568670308 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.02612436500021431, + "iterations": 1, + "max": 0.34897366800032614, + "mean": 0.3312113906002196, + "median": 0.327799070499168, + "min": 0.3119241430013062, + "rounds": 10, + "stddev": 0.013910482086376316 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.00480772399896523, + "iterations": 1, + "max": 0.12754467599916097, + "mean": 0.1221480985001108, + "median": 0.12221978500019759, + "min": 0.11572124899976188, + "rounds": 10, + "stddev": 0.003651635092442633 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.003961902998980804, + "iterations": 1, + "max": 0.09728806299972348, + "mean": 0.0825303886923883, + "median": 0.08013285700144479, + "min": 0.0780014840001968, + "rounds": 13, + "stddev": 0.0054482857115461455 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.046468831000311184, + "iterations": 1, + "max": 0.7463211720005347, + "mean": 0.6950176115999056, + "median": 0.6853093500003524, + "min": 0.6558438249994651, + "rounds": 10, + "stddev": 0.03193153158131865 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0037723280020145467, + "iterations": 1, + "max": 0.10660631399878184, + "mean": 0.10206309040004272, + "median": 0.10161684600006993, + "min": 0.0985671070011449, + "rounds": 10, + "stddev": 0.002512569282929824 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.005369793001591461, + "iterations": 1, + "max": 0.13290686599975743, + "mean": 0.11378327020029247, + "median": 0.11107024300054036, + "min": 0.10755836200041813, + "rounds": 10, + "stddev": 0.00728507513103704 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.006874631000755471, + "iterations": 1, + "max": 0.20964412600005744, + "mean": 0.19979873479987872, + "median": 0.1970382694998989, + "min": 0.19404061100067338, + "rounds": 10, + "stddev": 0.0054421122878378836 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.011623690499618533, + "iterations": 1, + "max": 0.12996738400033792, + "mean": 0.08845047399987986, + "median": 0.08316151350027212, + "min": 0.07914910499857797, + "rounds": 12, + "stddev": 0.014723623808893858 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.00940654099895255, + "iterations": 1, + "max": 0.09792000299967185, + "mean": 0.08880383863619169, + "median": 0.09074828699885984, + "min": 0.08132275800016942, + "rounds": 11, + "stddev": 0.0054859778466586516 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.026059696998345316, + "iterations": 1, + "max": 0.4623175329998048, + "mean": 0.434619928200118, + "median": 0.4321451744999649, + "min": 0.4172372369994264, + "rounds": 10, + "stddev": 0.014970537200223594 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.02200124399860215, + "iterations": 1, + "max": 0.5750828410000395, + "mean": 0.44501689140015516, + "median": 0.4319225429999278, + "min": 0.41361842400147, + "rounds": 10, + "stddev": 0.046990475469132875 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.005762115999459638, + "iterations": 1, + "max": 0.12046838300011586, + "mean": 0.10721757640003489, + "median": 0.10507006949865172, + "min": 0.09983395900053438, + "rounds": 10, + "stddev": 0.005944852322759856 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.005789047001599101, + "iterations": 1, + "max": 0.10347599099986837, + "mean": 0.098739981899962, + "median": 0.09923722500025178, + "min": 0.09399128699988069, + "rounds": 10, + "stddev": 0.003402453862533999 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.011289492000287282, + "iterations": 1, + "max": 0.21189056799994432, + "mean": 0.2035564803998568, + "median": 0.20206996549950418, + "min": 0.19445090200133563, + "rounds": 10, + "stddev": 0.006343425540922023 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0036781859998882283, + "iterations": 1, + "max": 0.08894176399917342, + "mean": 0.08210932699967088, + "median": 0.08154991250012245, + "min": 0.07848397199995816, + "rounds": 12, + "stddev": 0.003242889236063347 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.0058840259989665356, + "iterations": 1, + "max": 0.2407099100000778, + "mean": 0.23454736180010513, + "median": 0.23392340599912131, + "min": 0.22831901400058996, + "rounds": 10, + "stddev": 0.0036747222902471757 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.002072188000056485, + "iterations": 1, + "max": 0.08984946300006413, + "mean": 0.08494724183356084, + "median": 0.0842308730007062, + "min": 0.08327603799989447, + "rounds": 12, + "stddev": 0.0019672342192917484 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.012648214000364533, + "iterations": 1, + "max": 0.42045022000093013, + "mean": 0.4103736491002564, + "median": 0.4144640790000267, + "min": 0.3918969480000669, + "rounds": 10, + "stddev": 0.009178077995611465 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.013665847000083886, + "iterations": 1, + "max": 0.2824917390007613, + "mean": 0.26908035519991247, + "median": 0.2701626279995253, + "min": 0.25742512400029227, + "rounds": 10, + "stddev": 0.008458969862156886 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.008194876001653029, + "iterations": 1, + "max": 0.2038397440010158, + "mean": 0.19006024520003847, + "median": 0.1894416254999669, + "min": 0.17988157899890211, + "rounds": 10, + "stddev": 0.006923065255837666 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.1538464209988888, + "iterations": 1, + "max": 0.2844237230001454, + "mean": 0.195674094700189, + "median": 0.19519314050012326, + "min": 0.11517647800064879, + "rounds": 10, + "stddev": 0.0810306666261355 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.00347771249971629, + "iterations": 1, + "max": 0.08180292600081884, + "mean": 0.0797160748460066, + "median": 0.07939132699902984, + "min": 0.07666744600101083, + "rounds": 13, + "stddev": 0.001873638290223881 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.005855036000866676, + "iterations": 1, + "max": 0.11568180799986294, + "mean": 0.10772561599969777, + "median": 0.10647621649968642, + "min": 0.10181322999960685, + "rounds": 10, + "stddev": 0.0047422851004919895 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.007676460750190017, + "iterations": 1, + "max": 0.09144892199947208, + "mean": 0.08482993846141304, + "median": 0.08659366000028967, + "min": 0.07935997300046438, + "rounds": 13, + "stddev": 0.004262883138125922 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.005763257999205962, + "iterations": 1, + "max": 0.14944615800050087, + "mean": 0.1429630190003081, + "median": 0.141647902000841, + "min": 0.13607419900108653, + "rounds": 10, + "stddev": 0.003919677052911542 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.005203984750096424, + "iterations": 1, + "max": 0.08828753899979347, + "mean": 0.0818601455382752, + "median": 0.08086062999973365, + "min": 0.07738160400003835, + "rounds": 13, + "stddev": 0.003360566207830607 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.037146349999602535, + "iterations": 1, + "max": 0.7312834290005412, + "mean": 0.6851405824998438, + "median": 0.6841560580005535, + "min": 0.648478029999751, + "rounds": 10, + "stddev": 0.025393445800033807 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.027617137000561343, + "iterations": 1, + "max": 0.9301863350010535, + "mean": 0.7471751080998729, + "median": 0.7178547400008028, + "min": 0.6907707469999878, + "rounds": 10, + "stddev": 0.08206196410764437 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.018558206998932292, + "iterations": 1, + "max": 0.34662345000106143, + "mean": 0.32470166999974026, + "median": 0.3237635010000304, + "min": 0.30902637599865557, + "rounds": 10, + "stddev": 0.011680832293102504 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0033978619994741166, + "iterations": 1, + "max": 0.2557474019995425, + "mean": 0.0966439973002707, + "median": 0.07921657600036269, + "min": 0.07643111400102498, + "rounds": 10, + "stddev": 0.055932861457097076 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.003708141999595682, + "iterations": 1, + "max": 0.09411619999991672, + "mean": 0.08143365824995878, + "median": 0.07894791499984422, + "min": 0.07744693800123059, + "rounds": 12, + "stddev": 0.005397718612357221 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.011456876500233193, + "iterations": 1, + "max": 0.2771979030003422, + "mean": 0.10271868741665457, + "median": 0.08737897650007653, + "min": 0.07950401499874715, + "rounds": 12, + "stddev": 0.055281814004476085 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.004876026999227179, + "iterations": 1, + "max": 0.09678763699957926, + "mean": 0.08694906658365653, + "median": 0.08705959050075762, + "min": 0.08176763900155493, + "rounds": 12, + "stddev": 0.004034475505468624 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.007515919000070426, + "iterations": 1, + "max": 0.346149251998213, + "mean": 0.33774313759986396, + "median": 0.33839450900086376, + "min": 0.3275691020007798, + "rounds": 10, + "stddev": 0.005968396763588555 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.006131541002105223, + "iterations": 1, + "max": 0.4082648799994786, + "mean": 0.27338596749996213, + "median": 0.25986716150055145, + "min": 0.2462941049998335, + "rounds": 10, + "stddev": 0.047770797279175874 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.02492289899964817, + "iterations": 1, + "max": 1.0520390279998537, + "mean": 0.985836107099567, + "median": 1.0183273820002796, + "min": 0.6523057279991917, + "rounds": 10, + "stddev": 0.11850035525250827 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.7288956639986282, + "iterations": 1, + "max": 10.347908113000813, + "mean": 9.785941730500053, + "median": 9.922382428500896, + "min": 9.025855808000415, + "rounds": 10, + "stddev": 0.42129170851217657 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.020476330999372294, + "iterations": 1, + "max": 1.289848800000982, + "mean": 1.1765549573003227, + "median": 1.166366561000359, + "min": 1.1398172360004537, + "rounds": 10, + "stddev": 0.042875634848125566 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.22724668000046222, + "iterations": 1, + "max": 11.13369930600129, + "mean": 10.547195099400051, + "median": 10.548797896999531, + "min": 10.172394122000696, + "rounds": 10, + "stddev": 0.2619746473136364 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.478490536001118, + "iterations": 1, + "max": 1.7155814090001513, + "mean": 1.4502542314001403, + "median": 1.5283674885004075, + "min": 1.100909660999605, + "rounds": 10, + "stddev": 0.252268340144601 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.007751838998956373, + "iterations": 1, + "max": 0.46169115100019553, + "mean": 0.45127806949985827, + "median": 0.4524509550001312, + "min": 0.43585512099889456, + "rounds": 10, + "stddev": 0.008326200783659697 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.08950377300061518, + "iterations": 1, + "max": 12.870034137000403, + "mean": 12.750632888000291, + "median": 12.786243114999706, + "min": 12.526482204999411, + "rounds": 10, + "stddev": 0.10657015857200261 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.009829702001297846, + "iterations": 1, + "max": 0.45869010299975344, + "mean": 0.44652960440016615, + "median": 0.44848465399991255, + "min": 0.43295243800093886, + "rounds": 10, + "stddev": 0.008644899916523214 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.25738515600096434, + "iterations": 1, + "max": 0.965795927999352, + "mean": 0.7381892483001138, + "median": 0.7378135110002404, + "min": 0.4832729030003975, + "rounds": 10, + "stddev": 0.15477018320008754 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.7974443009989045, + "iterations": 1, + "max": 14.28242680800031, + "mean": 13.521334658800333, + "median": 13.28020127650052, + "min": 13.043321581999407, + "rounds": 10, + "stddev": 0.45884876190575324 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.11306887799946708, + "iterations": 1, + "max": 9.133826847000819, + "mean": 9.015908641099486, + "median": 9.025091175999478, + "min": 8.828873075999581, + "rounds": 10, + "stddev": 0.08717960974225451 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 287.2584820100001, + "mean": 287.2584820100001, + "median": 287.2584820100001, + "min": 287.2584820100001, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 644.9673524240006, + "mean": 644.9673524240006, + "median": 644.9673524240006, + "min": 644.9673524240006, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.014381830000274931, + "iterations": 1, + "max": 0.31679399099994043, + "mean": 0.3049286488998405, + "median": 0.30613645299945347, + "min": 0.28521640300095896, + "rounds": 10, + "stddev": 0.010296419320797515 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.029898084998421837, + "iterations": 1, + "max": 0.9064379089995782, + "mean": 0.7443077789002928, + "median": 0.736700457000552, + "min": 0.675791900999684, + "rounds": 10, + "stddev": 0.06237952846456562 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.14183163100096863, + "iterations": 1, + "max": 11.249002086000473, + "mean": 10.949170485500144, + "median": 10.929187951500353, + "min": 10.828263680999953, + "rounds": 10, + "stddev": 0.12213038025762499 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.46426850399984687, + "iterations": 1, + "max": 4.049499230000947, + "mean": 3.4919097714004237, + "median": 3.5393965340008435, + "min": 2.8217643209991365, + "rounds": 10, + "stddev": 0.34504615847597275 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.3259186499999487, + "iterations": 1, + "max": 2.7708884149997175, + "mean": 2.205849555999521, + "median": 2.2374005800002124, + "min": 1.686977929000932, + "rounds": 10, + "stddev": 0.32849005054152264 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.7318295610020868, + "iterations": 1, + "max": 30.190933095998844, + "mean": 28.231119643100282, + "median": 27.99763357700067, + "min": 27.305053561000022, + "rounds": 10, + "stddev": 0.9162891468935139 + } + }, + "started": "2021-12-19T00:16:35.031779+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-chatty.json new file mode 100644 index 00000000..7cd10f7c --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-18T23:23:15.395412+00:00", + "golden": "true", + "id": "run-2021-12-18T20_12_01_863_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 13613.400000000001, + "max": 44908.0, + "mean": 28922.146827826877, + "median": 34617.0, + "min": 1617.6666666666667, + "non_zero_mean": 28922.146827826877, + "non_zero_median": 34617.0, + "percentile25": 22529.066666666666, + "percentile75": 36142.46666666667, + "percentile90": 37119.693333333336, + "percentile99": 40837.65599999996, + "percentile999": 44500.96560000001, + "range": 43290.333333333336, + "samples": 69, + "stdev": 11699.547423847273, + "sum": 1995628.131120055 + } + }, + "cpu": { + "system": { + "iqr": 0.23533333333333528, + "max": 1.2879999999999958, + "mean": 1.0093042400886298, + "median": 1.208000000000004, + "min": 0.051333333333334966, + "non_zero_mean": 1.0093042400886298, + "non_zero_median": 1.208000000000004, + "percentile25": 0.9913333333333355, + "percentile75": 1.2266666666666708, + "percentile90": 1.2481333333333338, + "percentile99": 1.2748533333333305, + "percentile999": 1.2866853333333292, + "range": 1.2366666666666608, + "samples": 69, + "stdev": 0.3739527005223207, + "sum": 69.64199256611545 + }, + "user": { + "iqr": 1.8593333333333248, + "max": 6.313756856469751, + "mean": 4.950947113526402, + "median": 6.1693333333333165, + "min": 0.22333333333332725, + "non_zero_mean": 4.950947113526402, + "non_zero_median": 6.1693333333333165, + "percentile25": 4.3673333333333435, + "percentile75": 6.226666666666668, + "percentile90": 6.254933333333345, + "percentile99": 6.297290944875931, + "percentile999": 6.312110265310369, + "range": 6.090423523136423, + "samples": 69, + "stdev": 2.008417743366014, + "sum": 341.6153508333218 + } + }, + "entropy_available_bits": { + "iqr": 6.466897793186212, + "max": 217.46620450606585, + "mean": 26.451207474997315, + "median": 3.6, + "min": 0.0, + "non_zero_mean": 34.43647765612858, + "non_zero_median": 5.6, + "percentile25": 0.7333333333333333, + "percentile75": 7.2002311265195456, + "percentile90": 138.80000000000018, + "percentile99": 216.28785210860772, + "percentile999": 217.34836926632005, + "range": 217.46620450606585, + "samples": 69, + "stdev": 63.71597242357314, + "sum": 1825.1333157748147 + }, + "memory": { + "used": { + "iqr": 1534451712.0, + "max": 7565193216.0, + "mean": 6365891658.202899, + "median": 6835367936.0, + "min": 3476381696.0, + "non_zero_mean": 6365891658.202899, + "non_zero_median": 6835367936.0, + "percentile25": 5538103296.0, + "percentile75": 7072555008.0, + "percentile90": 7290961100.8, + "percentile99": 7510200647.679999, + "percentile999": 7559693959.168, + "range": 4088811520.0, + "samples": 69, + "stdev": 997331938.2998486, + "sum": 439246524416.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.005797101449275362, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2, + "non_zero_median": 0.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.1759999999999991, + "percentile999": 0.2576000000000003, + "range": 0.26666666666666666, + "samples": 69, + "stdev": 0.0356803719953032, + "sum": 0.4 + } + }, + "writes_completed": { + "total": { + "iqr": 2466.5999999999995, + "max": 4767.825752827966, + "mean": 3308.5428701144237, + "median": 4392.666666666666, + "min": 0.7999999999999999, + "non_zero_mean": 3308.5428701144237, + "non_zero_median": 4392.666666666666, + "percentile25": 2046.1999999999998, + "percentile75": 4512.799999999999, + "percentile90": 4621.4484403262, + "percentile99": 4766.538907571616, + "percentile999": 4767.697068302331, + "range": 4767.025752827966, + "samples": 69, + "stdev": 1664.1346383641505, + "sum": 228289.45803789518 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 245.0666666666666, + "max": 2211.266666666667, + "mean": 1523.9980676328503, + "median": 1573.9333333333334, + "min": 554.2, + "non_zero_mean": 1523.9980676328503, + "non_zero_median": 1573.9333333333334, + "percentile25": 1447.6666666666667, + "percentile75": 1692.7333333333333, + "percentile90": 1766.6533333333332, + "percentile99": 2002.007999999998, + "percentile999": 2190.340800000001, + "range": 1657.0666666666668, + "samples": 69, + "stdev": 296.75661072549497, + "sum": 105155.86666666671 + } + }, + "cpu": { + "system": { + "iqr": 0.03200000000000022, + "max": 0.28133333333333327, + "mean": 0.15513043478260868, + "median": 0.16533333333333264, + "min": 0.024666666666668866, + "non_zero_mean": 0.15513043478260868, + "non_zero_median": 0.16533333333333264, + "percentile25": 0.14533333333333381, + "percentile75": 0.17733333333333404, + "percentile90": 0.2037333333333328, + "percentile99": 0.2405333333333326, + "percentile999": 0.27725333333333335, + "range": 0.2566666666666644, + "samples": 69, + "stdev": 0.04845296949530281, + "sum": 10.703999999999999 + }, + "user": { + "iqr": 0.0626666666666656, + "max": 0.5300000000000007, + "mean": 0.2145990338164251, + "median": 0.1753333333333316, + "min": 0.03133333333333231, + "non_zero_mean": 0.2145990338164251, + "non_zero_median": 0.1753333333333316, + "percentile25": 0.16400000000000053, + "percentile75": 0.22666666666666613, + "percentile90": 0.44239999999999974, + "percentile99": 0.5068800000000006, + "percentile999": 0.5276880000000008, + "range": 0.49866666666666837, + "samples": 69, + "stdev": 0.11846988227219926, + "sum": 14.80733333333333 + } + }, + "entropy_available_bits": { + "iqr": 0.2666666666666666, + "max": 201.73333333333332, + "mean": 12.155555555555555, + "median": 1.2666666666666666, + "min": 0.3333333333333333, + "non_zero_mean": 12.155555555555555, + "non_zero_median": 1.2666666666666666, + "percentile25": 1.2, + "percentile75": 1.4666666666666666, + "percentile90": 2.053333333333334, + "percentile99": 195.56799999999993, + "percentile999": 201.1168, + "range": 201.39999999999998, + "samples": 69, + "stdev": 44.1206866345794, + "sum": 838.7333333333336 + }, + "memory": { + "used": { + "iqr": 31531008.0, + "max": 553680896.0, + "mean": 460627730.5507246, + "median": 465317888.0, + "min": 372219904.0, + "non_zero_mean": 460627730.5507246, + "non_zero_median": 465317888.0, + "percentile25": 440500224.0, + "percentile75": 472031232.0, + "percentile90": 533911961.6, + "percentile99": 550998671.36, + "percentile999": 553412673.536, + "range": 181460992.0, + "samples": 69, + "stdev": 44912617.058856696, + "sum": 31783313408.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1588701.8666666667, + "max": 3334144.0, + "mean": 733939.8801932367, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1947763.5282051282, + "non_zero_median": 2044177.0666666667, + "percentile25": 0.0, + "percentile75": 1588701.8666666667, + "percentile90": 2335921.4933333346, + "percentile99": 3326345.216, + "percentile999": 3333364.1216, + "range": 3334144.0, + "samples": 69, + "stdev": 1113799.4197610859, + "sum": 50641851.733333334 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.002192, + "max": 4.763156, + "mean": 0.7430196376811594, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.8482419444444442, + "non_zero_median": 4.564477999999999, + "percentile25": 0.0, + "percentile75": 0.002192, + "percentile90": 4.613866600000001, + "percentile99": 4.75577256, + "percentile999": 4.762417656, + "range": 4.763156, + "samples": 69, + "stdev": 1.7139272591857908, + "sum": 51.26835500000001 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 1.401038, + "mean": 0.03930028985507246, + "median": 0.0, + "min": -0.000397, + "non_zero_mean": 0.1506511111111111, + "non_zero_median": 0.010877, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.015403200000000008, + "percentile99": 0.8529505199999945, + "percentile999": 1.3462292520000017, + "range": 1.401435, + "samples": 69, + "stdev": 0.18641944528031573, + "sum": 2.7117199999999997 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.8, + "mean": 0.15845410628019324, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.6444444444444444, + "non_zero_median": 4.733333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.754666666666666, + "percentile999": 4.795466666666667, + "range": 4.8, + "samples": 69, + "stdev": 0.8195408765103112, + "sum": 10.933333333333334 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9986.066666666668, + "mean": 433.736231884058, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9975.933333333334, + "non_zero_median": 9985.666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9985.794666666667, + "percentile999": 9986.039466666667, + "range": 9986.066666666668, + "samples": 69, + "stdev": 2049.3096345420254, + "sum": 29927.800000000003 + }, + "pg_stat_database_blks_hit": { + "iqr": 2147.1333333333314, + "max": 36568.333333333336, + "mean": 19822.340096618358, + "median": 20916.866666666665, + "min": 5359.866666666667, + "non_zero_mean": 19822.340096618358, + "non_zero_median": 20916.866666666665, + "percentile25": 19474.466666666667, + "percentile75": 21621.6, + "percentile90": 24538.386666666665, + "percentile99": 35361.28799999999, + "percentile999": 36447.628800000006, + "range": 31208.466666666667, + "samples": 69, + "stdev": 5743.834228156469, + "sum": 1367741.466666667 + }, + "pg_stat_database_blks_read": { + "iqr": 65.0, + "max": 152.93333333333334, + "mean": 93.23574879227053, + "median": 120.0, + "min": 0.0, + "non_zero_mean": 94.60686274509804, + "non_zero_median": 120.16666666666666, + "percentile25": 62.8, + "percentile75": 127.8, + "percentile90": 134.30666666666667, + "percentile99": 152.16266666666667, + "percentile999": 152.85626666666667, + "range": 152.93333333333334, + "samples": 69, + "stdev": 48.05436406659091, + "sum": 6433.266666666666 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 69, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 65.0, + "max": 152.93333333333334, + "mean": 93.23574879227053, + "median": 120.0, + "min": 0.0, + "non_zero_mean": 94.60686274509804, + "non_zero_median": 120.16666666666666, + "percentile25": 62.8, + "percentile75": 127.8, + "percentile90": 134.30666666666667, + "percentile99": 152.16266666666667, + "percentile999": 152.85626666666667, + "range": 152.93333333333334, + "samples": 69, + "stdev": 48.05436406659091, + "sum": 6433.266666666666 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.6, + "mean": 0.18357487922705315, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8444444444444444, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.4186666666666647, + "percentile999": 1.5818666666666674, + "range": 1.6, + "samples": 69, + "stdev": 0.3959609469588249, + "sum": 12.666666666666666 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2201.0666666666675, + "max": 23544.4, + "mean": 6410.269565217392, + "median": 4731.2, + "min": 2760.733333333333, + "non_zero_mean": 6410.269565217392, + "non_zero_median": 4731.2, + "percentile25": 4357.466666666666, + "percentile75": 6558.533333333334, + "percentile90": 11373.480000000001, + "percentile99": 22341.570666666656, + "percentile999": 23424.117066666673, + "range": 20783.666666666668, + "samples": 69, + "stdev": 3946.880143875944, + "sum": 442308.6000000001 + }, + "pg_stat_database_tup_inserted": { + "iqr": 352.3333333333333, + "max": 852.5333333333333, + "mean": 496.08115942028985, + "median": 635.9333333333333, + "min": 0.0, + "non_zero_mean": 503.3764705882353, + "non_zero_median": 643.3333333333333, + "percentile25": 326.2, + "percentile75": 678.5333333333333, + "percentile90": 710.7733333333333, + "percentile99": 827.373333333333, + "percentile999": 850.0173333333333, + "range": 852.5333333333333, + "samples": 69, + "stdev": 255.6644039242266, + "sum": 34229.600000000006 + }, + "pg_stat_database_tup_returned": { + "iqr": 2447.4000000000005, + "max": 27896.866666666665, + "mean": 8179.060869565217, + "median": 6057.266666666666, + "min": 4179.266666666666, + "non_zero_mean": 8179.060869565217, + "non_zero_median": 6057.266666666666, + "percentile25": 5579.4, + "percentile75": 8026.8, + "percentile90": 15773.746666666668, + "percentile99": 27632.84533333333, + "percentile999": 27870.464533333332, + "range": 23717.6, + "samples": 69, + "stdev": 5064.645243907274, + "sum": 564355.2000000001 + }, + "pg_stat_database_tup_updated": { + "iqr": 5.333333333333334, + "max": 136.66666666666666, + "mean": 9.479227053140097, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 11.08587570621469, + "non_zero_median": 1.0666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 5.4, + "percentile90": 19.30666666666667, + "percentile99": 128.00799999999992, + "percentile999": 135.8008, + "range": 136.66666666666666, + "samples": 69, + "stdev": 25.92790475810572, + "sum": 654.0666666666665 + }, + "pg_stat_database_xact_commit": { + "iqr": 9.600000000000001, + "max": 123.93333333333334, + "mean": 52.62995169082126, + "median": 44.86666666666667, + "min": 6.2, + "non_zero_mean": 52.62995169082126, + "non_zero_median": 44.86666666666667, + "percentile25": 42.8, + "percentile75": 52.4, + "percentile90": 110.28, + "percentile99": 122.98133333333332, + "percentile999": 123.83813333333335, + "range": 117.73333333333333, + "samples": 69, + "stdev": 26.13581804972538, + "sum": 3631.466666666667 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 69, + "stdev": 0.0, + "sum": 4.600000000000003 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003864734299516908, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.08533333333333151, + "percentile999": 0.2485333333333339, + "range": 0.26666666666666666, + "samples": 69, + "stdev": 0.032102894156205125, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 23.799999999999997, + "max": 158.06666666666666, + "mean": 68.35169082125604, + "median": 65.80000000000001, + "min": 5.866666666666667, + "non_zero_mean": 68.35169082125604, + "non_zero_median": 65.80000000000001, + "percentile25": 58.266666666666666, + "percentile75": 82.06666666666666, + "percentile90": 93.08, + "percentile99": 145.82666666666654, + "percentile999": 156.8426666666667, + "range": 152.2, + "samples": 69, + "stdev": 25.095144138936185, + "sum": 4716.266666666668 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "4,5,3,6,8,7,10,9,11,14,19,21,23,22,18,20,25,24,26,27,30,29,34,33,28,31,32,35,36,37,38,39,40,42,41,45,44,43,46,47,48,49,50,51,52,54,53,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-18T23:03:17.177902Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2021-12-18T23:03:18.371597Z 2021-12-18T23:03:18.359286Z 2021-12-18T23:03:18.355045Z 2021-12-18T23:03:17.436638Z 2021-12-18T23:03:17.343671Z 2021-12-18T23:03:17.253268Z 2021-12-18T23:03:17.177902Z 2021-12-18T23:03:19.478411Z 2021-12-18T23:03:18.521174Z 2021-12-18T23:03:18.412029Z 2021-12-18T23:07:51.590756Z 2021-12-18T23:07:50.829940Z 2021-12-18T23:07:50.339692Z 2021-12-18T23:07:50.328991Z 2021-12-18T23:07:50.279954Z 2021-12-18T23:07:50.215984Z 2021-12-18T23:07:50.199068Z 2021-12-18T23:07:50.190995Z 2021-12-18T23:07:50.191885Z 2021-12-18T23:07:50.190973Z 2021-12-18T23:11:06.095042Z 2021-12-18T23:11:05.822746Z 2021-12-18T23:11:05.752364Z 2021-12-18T23:11:05.181864Z 2021-12-18T23:11:05.181416Z 2021-12-18T23:11:05.170319Z 2021-12-18T23:11:05.155133Z 2021-12-18T23:11:05.097806Z 2021-12-18T23:11:05.095654Z 2021-12-18T23:11:05.084389Z 2021-12-18T23:14:14.785290Z 2021-12-18T23:14:14.731482Z 2021-12-18T23:14:14.334839Z 2021-12-18T23:14:14.322269Z 2021-12-18T23:14:14.300413Z 2021-12-18T23:14:14.288547Z 2021-12-18T23:14:14.239153Z 2021-12-18T23:14:14.230814Z 2021-12-18T23:14:14.185609Z 2021-12-18T23:14:14.068344Z 2021-12-18T23:17:35.378672Z 2021-12-18T23:17:35.283155Z 2021-12-18T23:17:34.821761Z 2021-12-18T23:17:34.816852Z 2021-12-18T23:17:34.814173Z 2021-12-18T23:17:34.812400Z 2021-12-18T23:17:34.738532Z 2021-12-18T23:17:34.675216Z 2021-12-18T23:17:34.675223Z 2021-12-18T23:17:34.660714Z", + "created_diff": 1.2291698000000002, + "ended": "2021-12-18T23:20:19.429372Z", + "error_job_ids": " []", + "event_first": "2021-12-18T23:03:29.306314Z 2021-12-18T23:05:09.863658Z 2021-12-18T23:03:29.966473Z 2021-12-18T23:04:33.504735Z 2021-12-18T23:03:23.716564Z 2021-12-18T23:04:09.438535Z 2021-12-18T23:03:24.970541Z 2021-12-18T23:04:47.865594Z 2021-12-18T23:03:33.431291Z 2021-12-18T23:04:20.756340Z 2021-12-18T23:08:07.843588Z 2021-12-18T23:08:07.160944Z 2021-12-18T23:08:01.070999Z 2021-12-18T23:08:03.490324Z 2021-12-18T23:07:57.511554Z 2021-12-18T23:07:59.150480Z 2021-12-18T23:07:58.003659Z 2021-12-18T23:07:58.015666Z 2021-12-18T23:07:59.704384Z 2021-12-18T23:07:58.017657Z 2021-12-18T23:11:22.085084Z 2021-12-18T23:11:19.697872Z 2021-12-18T23:11:17.383168Z 2021-12-18T23:11:16.233328Z 2021-12-18T23:11:15.203793Z 2021-12-18T23:11:12.834438Z 2021-12-18T23:11:12.427538Z 2021-12-18T23:11:12.317982Z 2021-12-18T23:11:11.649069Z 2021-12-18T23:11:14.749772Z 2021-12-18T23:14:46.485882Z 2021-12-18T23:14:49.795425Z 2021-12-18T23:14:22.664083Z 2021-12-18T23:14:22.575374Z 2021-12-18T23:14:22.664083Z 2021-12-18T23:14:22.975377Z 2021-12-18T23:14:20.631619Z 2021-12-18T23:14:20.665380Z 2021-12-18T23:14:20.457648Z 2021-12-18T23:14:20.712107Z 2021-12-18T23:17:51.553135Z 2021-12-18T23:17:49.694712Z 2021-12-18T23:17:46.078587Z 2021-12-18T23:17:44.918528Z 2021-12-18T23:17:42.153645Z 2021-12-18T23:17:41.967125Z 2021-12-18T23:17:44.017656Z 2021-12-18T23:17:41.651365Z 2021-12-18T23:17:44.733481Z 2021-12-18T23:17:42.185903Z", + "event_last": "2021-12-18T23:06:09.435718Z 2021-12-18T23:07:22.135633Z 2021-12-18T23:06:08.683247Z 2021-12-18T23:06:59.213200Z 2021-12-18T23:06:03.342597Z 2021-12-18T23:06:43.064590Z 2021-12-18T23:06:04.976901Z 2021-12-18T23:07:08.492931Z 2021-12-18T23:06:12.946724Z 2021-12-18T23:06:52.000531Z 2021-12-18T23:10:34.617611Z 2021-12-18T23:10:35.077541Z 2021-12-18T23:10:29.220220Z 2021-12-18T23:10:34.943542Z 2021-12-18T23:10:27.366065Z 2021-12-18T23:10:34.172674Z 2021-12-18T23:10:27.882393Z 2021-12-18T23:10:28.418654Z 2021-12-18T23:10:33.836149Z 2021-12-18T23:10:33.291605Z 2021-12-18T23:13:50.579687Z 2021-12-18T23:13:46.848675Z 2021-12-18T23:13:44.833029Z 2021-12-18T23:13:45.670198Z 2021-12-18T23:13:43.684163Z 2021-12-18T23:13:44.313714Z 2021-12-18T23:13:42.325748Z 2021-12-18T23:13:44.584767Z 2021-12-18T23:13:39.390034Z 2021-12-18T23:13:45.303528Z 2021-12-18T23:17:08.533974Z 2021-12-18T23:17:09.568211Z 2021-12-18T23:16:53.686329Z 2021-12-18T23:16:54.707049Z 2021-12-18T23:16:52.998881Z 2021-12-18T23:16:54.639642Z 2021-12-18T23:16:52.120966Z 2021-12-18T23:16:53.405152Z 2021-12-18T23:16:52.043655Z 2021-12-18T23:16:53.876530Z 2021-12-18T23:20:19.276531Z 2021-12-18T23:20:19.429372Z 2021-12-18T23:20:16.843813Z 2021-12-18T23:20:17.090704Z 2021-12-18T23:20:13.620686Z 2021-12-18T23:20:15.940875Z 2021-12-18T23:20:14.847091Z 2021-12-18T23:20:13.842191Z 2021-12-18T23:20:16.573508Z 2021-12-18T23:20:16.362422Z", + "failed_job_ids": " []", + "finished": "2021-12-18T23:06:13.765329Z 2021-12-18T23:07:24.413785Z 2021-12-18T23:06:12.186233Z 2021-12-18T23:07:02.961823Z 2021-12-18T23:06:05.639518Z 2021-12-18T23:06:46.687028Z 2021-12-18T23:06:08.677318Z 2021-12-18T23:07:12.590486Z 2021-12-18T23:06:17.225637Z 2021-12-18T23:06:55.451626Z 2021-12-18T23:10:38.424798Z 2021-12-18T23:10:38.726995Z 2021-12-18T23:10:33.524805Z 2021-12-18T23:10:37.485953Z 2021-12-18T23:10:30.935189Z 2021-12-18T23:10:34.205508Z 2021-12-18T23:10:31.860537Z 2021-12-18T23:10:31.887096Z 2021-12-18T23:10:33.918519Z 2021-12-18T23:10:33.652516Z 2021-12-18T23:13:52.199274Z 2021-12-18T23:13:51.298455Z 2021-12-18T23:13:48.800515Z 2021-12-18T23:13:49.025094Z 2021-12-18T23:13:47.593447Z 2021-12-18T23:13:44.806538Z 2021-12-18T23:13:46.438981Z 2021-12-18T23:13:45.436838Z 2021-12-18T23:13:43.107952Z 2021-12-18T23:13:47.268815Z 2021-12-18T23:17:12.175194Z 2021-12-18T23:17:13.843957Z 2021-12-18T23:16:57.177356Z 2021-12-18T23:16:58.270975Z 2021-12-18T23:16:56.679860Z 2021-12-18T23:16:57.673070Z 2021-12-18T23:16:56.337989Z 2021-12-18T23:16:55.357036Z 2021-12-18T23:16:56.184603Z 2021-12-18T23:16:56.383793Z 2021-12-18T23:20:23.704520Z 2021-12-18T23:20:22.767443Z 2021-12-18T23:20:20.610563Z 2021-12-18T23:20:19.663292Z 2021-12-18T23:20:17.355762Z 2021-12-18T23:20:17.352829Z 2021-12-18T23:20:18.456085Z 2021-12-18T23:20:18.080711Z 2021-12-18T23:20:18.249673Z 2021-12-18T23:20:17.947573Z", + "finished_diff": 24.0992014, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 244.379199, + "mean": 168.88623489999995, + "min": 155.779558 + }, + "jobs_events_duration": { + "max": 160.129404, + "mean": 150.68214384000004, + "min": 132.271975 + }, + "jobs_events_lag": { + "max": -0.032834, + "mean": -3.0881848199999995, + "min": -4.44978 + }, + "jobs_waiting": { + "max": 21.610346, + "mean": 2.38579394, + "min": 1.026604 + }, + "modified": "2021-12-18T23:03:19.576762Z 2021-12-18T23:03:19.537765Z 2021-12-18T23:03:19.499775Z 2021-12-18T23:03:17.995240Z 2021-12-18T23:03:17.962413Z 2021-12-18T23:03:17.928770Z 2021-12-18T23:03:17.894663Z 2021-12-18T23:03:29.428870Z 2021-12-18T23:03:19.659519Z 2021-12-18T23:03:19.622109Z 2021-12-18T23:07:52.828303Z 2021-12-18T23:07:52.760655Z 2021-12-18T23:07:51.022398Z 2021-12-18T23:07:50.989980Z 2021-12-18T23:07:50.957966Z 2021-12-18T23:07:50.920445Z 2021-12-18T23:07:50.884613Z 2021-12-18T23:07:50.819324Z 2021-12-18T23:07:50.852174Z 2021-12-18T23:07:50.781431Z 2021-12-18T23:11:07.164312Z 2021-12-18T23:11:07.127615Z 2021-12-18T23:11:07.089095Z 2021-12-18T23:11:05.924947Z 2021-12-18T23:11:05.891621Z 2021-12-18T23:11:05.854608Z 2021-12-18T23:11:05.819688Z 2021-12-18T23:11:05.786737Z 2021-12-18T23:11:05.753018Z 2021-12-18T23:11:05.718129Z 2021-12-18T23:14:35.602849Z 2021-12-18T23:14:35.468699Z 2021-12-18T23:14:14.992797Z 2021-12-18T23:14:14.958465Z 2021-12-18T23:14:14.925671Z 2021-12-18T23:14:14.889098Z 2021-12-18T23:14:14.857050Z 2021-12-18T23:14:14.820123Z 2021-12-18T23:14:14.783419Z 2021-12-18T23:14:14.745092Z 2021-12-18T23:17:36.804496Z 2021-12-18T23:17:36.719427Z 2021-12-18T23:17:35.503598Z 2021-12-18T23:17:35.469218Z 2021-12-18T23:17:35.435931Z 2021-12-18T23:17:35.402528Z 2021-12-18T23:17:35.365590Z 2021-12-18T23:17:35.300731Z 2021-12-18T23:17:35.332685Z 2021-12-18T23:17:35.268172Z", + "modified_diff": 7.4842686, + "started": "2021-12-18T23:03:20.120420Z 2021-12-18T23:03:20.034586Z 2021-12-18T23:03:19.952996Z 2021-12-18T23:03:18.506026Z 2021-12-18T23:03:18.385030Z 2021-12-18T23:03:18.369742Z 2021-12-18T23:03:18.204506Z 2021-12-18T23:03:29.625705Z 2021-12-18T23:03:20.388143Z 2021-12-18T23:03:20.220443Z 2021-12-18T23:07:53.248167Z 2021-12-18T23:07:53.112196Z 2021-12-18T23:07:52.019728Z 2021-12-18T23:07:51.928803Z 2021-12-18T23:07:51.780151Z 2021-12-18T23:07:51.671852Z 2021-12-18T23:07:51.600190Z 2021-12-18T23:07:51.420407Z 2021-12-18T23:07:51.492321Z 2021-12-18T23:07:51.294439Z 2021-12-18T23:11:07.730143Z 2021-12-18T23:11:07.657007Z 2021-12-18T23:11:07.485030Z 2021-12-18T23:11:06.663741Z 2021-12-18T23:11:06.552613Z 2021-12-18T23:11:06.464060Z 2021-12-18T23:11:06.353020Z 2021-12-18T23:11:06.282824Z 2021-12-18T23:11:06.192344Z 2021-12-18T23:11:06.115651Z 2021-12-18T23:14:36.395636Z 2021-12-18T23:14:36.249003Z 2021-12-18T23:14:15.819184Z 2021-12-18T23:14:15.769865Z 2021-12-18T23:14:15.594305Z 2021-12-18T23:14:15.522245Z 2021-12-18T23:14:15.409737Z 2021-12-18T23:14:15.330492Z 2021-12-18T23:14:15.248560Z 2021-12-18T23:14:15.161312Z 2021-12-18T23:17:37.511193Z 2021-12-18T23:17:37.345425Z 2021-12-18T23:17:36.359314Z 2021-12-18T23:17:36.197705Z 2021-12-18T23:17:36.107735Z 2021-12-18T23:17:36.016623Z 2021-12-18T23:17:35.931552Z 2021-12-18T23:17:35.771730Z 2021-12-18T23:17:35.851805Z 2021-12-18T23:17:35.691442Z", + "started_diff": 7.608698799999999, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-18T23:23:12.126606+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-launching.json new file mode 100644 index 00000000..77d3c006 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-18T20_12_01_863_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-19T00:15:40.792429+00:00", + "golden": "true", + "id": "run-2021-12-18T20_12_01_863_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 16325.23333333333, + "max": 28763.800000000003, + "mean": 9611.145705824003, + "median": 4581.8, + "min": 1386.2666666666667, + "non_zero_mean": 9611.145705824003, + "non_zero_median": 4581.8, + "percentile25": 1787.4333333333334, + "percentile75": 18112.666666666664, + "percentile90": 21725.266666666666, + "percentile99": 28445.379999999997, + "percentile999": 28731.958000000006, + "range": 27377.533333333336, + "samples": 91, + "stdev": 8752.00063270727, + "sum": 874614.2592299839 + } + }, + "cpu": { + "system": { + "iqr": 1.6886666666666759, + "max": 2.4340000000000073, + "mean": 0.8741470808306468, + "median": 0.3153333333333364, + "min": 0.03400000000000129, + "non_zero_mean": 0.8741470808306468, + "non_zero_median": 0.3153333333333364, + "percentile25": 0.053666666666661436, + "percentile75": 1.7423333333333373, + "percentile90": 2.1346666666666656, + "percentile99": 2.308000000000008, + "percentile999": 2.421400000000009, + "range": 2.400000000000006, + "samples": 91, + "stdev": 0.8789531232152512, + "sum": 79.54738435558883 + }, + "user": { + "iqr": 5.289669333155557, + "max": 6.123333333333313, + "mean": 2.7781459808942537, + "median": 1.2773333333333161, + "min": 0.10800000000000788, + "non_zero_mean": 2.7781459808942537, + "non_zero_median": 1.2773333333333161, + "percentile25": 0.3209973335110953, + "percentile75": 5.610666666666653, + "percentile90": 5.960000000000014, + "percentile99": 6.097533333333322, + "percentile999": 6.120753333333314, + "range": 6.015333333333305, + "samples": 91, + "stdev": 2.5588432063730067, + "sum": 252.8112842613771 + } + }, + "entropy_available_bits": { + "iqr": 4.000000000000001, + "max": 210.9859653800858, + "mean": 20.339040988118175, + "median": 1.4666666666666666, + "min": 0.0, + "non_zero_mean": 25.706287915538248, + "non_zero_median": 3.2, + "percentile25": 0.3333333333333333, + "percentile75": 4.333333333333334, + "percentile90": 5.466666666666667, + "percentile99": 210.93859653800857, + "percentile999": 210.98122849587807, + "range": 210.9859653800858, + "samples": 91, + "stdev": 59.5024184750913, + "sum": 1850.852729918754 + }, + "memory": { + "used": { + "iqr": 5705822208.0, + "max": 17292341248.0, + "mean": 8955609268.043957, + "median": 6773239808.0, + "min": 5305671680.0, + "non_zero_mean": 8955609268.043957, + "non_zero_median": 6773239808.0, + "percentile25": 6327810048.0, + "percentile75": 12033632256.0, + "percentile90": 15056883712.0, + "percentile99": 17223822131.2, + "percentile999": 17285489336.32, + "range": 11986669568.0, + "samples": 91, + "stdev": 3624828599.84529, + "sum": 814960443392.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.0029304029304029304, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.02666666666666515, + "percentile999": 0.24266666666666953, + "range": 0.26666666666666666, + "samples": 91, + "stdev": 0.027954262312584487, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 828.4000000000001, + "max": 1517.4, + "mean": 420.28739304476125, + "median": 75.4, + "min": 0.26666666666666666, + "non_zero_mean": 420.28739304476125, + "non_zero_median": 75.4, + "percentile25": 2.7666666666666666, + "percentile75": 831.1666666666667, + "percentile90": 1264.7734204502522, + "percentile99": 1517.1000000000001, + "percentile999": 1517.3700000000001, + "range": 1517.1333333333334, + "samples": 91, + "stdev": 516.7502961108491, + "sum": 38246.15276707326 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2119.133333333333, + "max": 3574.733333333333, + "mean": 1628.7238095238095, + "median": 1542.3333333333333, + "min": 409.0, + "non_zero_mean": 1628.7238095238095, + "non_zero_median": 1542.3333333333333, + "percentile25": 604.5, + "percentile75": 2723.633333333333, + "percentile90": 3004.8, + "percentile99": 3510.533333333333, + "percentile999": 3568.313333333334, + "range": 3165.733333333333, + "samples": 91, + "stdev": 1063.7984587536123, + "sum": 148213.86666666664 + } + }, + "cpu": { + "system": { + "iqr": 0.25000000000000006, + "max": 0.5039999999999982, + "mean": 0.1586080586080586, + "median": 0.12999999999999923, + "min": 0.013999999999998639, + "non_zero_mean": 0.1586080586080586, + "non_zero_median": 0.12999999999999923, + "percentile25": 0.028666666666665223, + "percentile75": 0.2786666666666653, + "percentile90": 0.3473333333333339, + "percentile99": 0.4698000000000019, + "percentile999": 0.500579999999999, + "range": 0.4899999999999996, + "samples": 91, + "stdev": 0.14260800622864803, + "sum": 14.433333333333332 + }, + "user": { + "iqr": 0.3130000000000005, + "max": 0.875333333333333, + "mean": 0.2065054945054945, + "median": 0.12999999999999923, + "min": 0.012666666666666517, + "non_zero_mean": 0.2065054945054945, + "non_zero_median": 0.12999999999999923, + "percentile25": 0.04000000000000057, + "percentile75": 0.3530000000000011, + "percentile90": 0.450666666666668, + "percentile99": 0.735533333333333, + "percentile999": 0.8613533333333347, + "range": 0.8626666666666665, + "samples": 91, + "stdev": 0.19738781381723924, + "sum": 18.792000000000005 + } + }, + "entropy_available_bits": { + "iqr": 2.833333333333333, + "max": 210.66666666666666, + "mean": 10.317216117216116, + "median": 0.3333333333333333, + "min": 0.0, + "non_zero_mean": 17.386419753086418, + "non_zero_median": 1.9333333333333336, + "percentile25": 0.0, + "percentile75": 2.833333333333333, + "percentile90": 3.933333333333333, + "percentile99": 210.66666666666666, + "percentile999": 210.66666666666666, + "range": 210.66666666666666, + "samples": 91, + "stdev": 43.22385525696426, + "sum": 938.8666666666666 + }, + "memory": { + "used": { + "iqr": 379533312.0, + "max": 1087942656.0, + "mean": 576688082.9890109, + "median": 406364160.0, + "min": 368242688.0, + "non_zero_mean": 576688082.9890109, + "non_zero_median": 406364160.0, + "percentile25": 380473344.0, + "percentile75": 760006656.0, + "percentile90": 991334400.0, + "percentile99": 1082519961.6, + "percentile999": 1087400386.5600002, + "range": 719699968.0, + "samples": 91, + "stdev": 247164686.7347254, + "sum": 52478615552.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 3549.866666666667, + "max": 55504681.53333333, + "mean": 624038.997069597, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2366147.8638888886, + "non_zero_median": 52974.933333333334, + "percentile25": 0.0, + "percentile75": 3549.866666666667, + "percentile90": 59528.53333333333, + "percentile99": 5687602.233333019, + "percentile999": 50522973.60333393, + "range": 55504681.53333333, + "samples": 91, + "stdev": 5817064.9333553435, + "sum": 56787548.73333334 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 2.59178, + "mean": 0.07220030769230769, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.312868, + "non_zero_median": 0.02078, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.02172, + "percentile99": 1.789709899999995, + "percentile999": 2.5115729900000097, + "range": 2.59178, + "samples": 91, + "stdev": 0.33907723803744816, + "sum": 6.570227999999999 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.5030835, + "max": 13.015151, + "mean": 0.6222932417582417, + "median": 0.0, + "min": -0.000922, + "non_zero_mean": 1.3483020238095238, + "non_zero_median": 0.8344045, + "percentile25": 0.0, + "percentile75": 0.5030835, + "percentile90": 1.373189, + "percentile99": 7.769418199999967, + "percentile999": 12.490577720000063, + "range": 13.016072999999999, + "samples": 91, + "stdev": 1.684777738946109, + "sum": 56.628685 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.066666666666666, + "mean": 0.17875457875457876, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.2533333333333334, + "non_zero_median": 3.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.8866666666666654, + "percentile999": 4.048666666666668, + "range": 4.066666666666666, + "samples": 91, + "stdev": 0.7586784559963461, + "sum": 16.266666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9995.466666666667, + "mean": 493.5868131868132, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8983.28, + "non_zero_median": 9991.266666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9994.866666666667, + "percentile999": 9995.406666666668, + "range": 9995.466666666667, + "samples": 91, + "stdev": 2081.82365784009, + "sum": 44916.4 + }, + "pg_stat_database_blks_hit": { + "iqr": 22202.23333333333, + "max": 98010.53333333334, + "mean": 19207.28205128205, + "median": 8505.666666666666, + "min": 1167.8666666666666, + "non_zero_mean": 19207.28205128205, + "non_zero_median": 8505.666666666666, + "percentile25": 6565.366666666667, + "percentile75": 28767.6, + "percentile90": 41468.6, + "percentile99": 75374.99333333319, + "percentile999": 95746.97933333361, + "range": 96842.66666666667, + "samples": 91, + "stdev": 18737.111763393103, + "sum": 1747862.6666666674 + }, + "pg_stat_database_blks_read": { + "iqr": 2.533333333333333, + "max": 11.0, + "mean": 1.8131868131868132, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.0555555555555554, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 2.533333333333333, + "percentile90": 6.4, + "percentile99": 9.67999999999999, + "percentile999": 10.868000000000016, + "range": 11.0, + "samples": 91, + "stdev": 2.630124059980043, + "sum": 165.00000000000003 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 91, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 2.533333333333333, + "max": 11.0, + "mean": 1.8131868131868132, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.0555555555555554, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 2.533333333333333, + "percentile90": 6.4, + "percentile99": 9.67999999999999, + "percentile999": 10.868000000000016, + "range": 11.0, + "samples": 91, + "stdev": 2.630124059980043, + "sum": 165.00000000000003 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.2666666666666666, + "mean": 0.16703296703296702, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.95, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9333333333333333, + "percentile99": 1.2666666666666666, + "percentile999": 1.2666666666666666, + "range": 1.2666666666666666, + "samples": 91, + "stdev": 0.3854609075467678, + "sum": 15.199999999999998 + }, + "pg_stat_database_tup_fetched": { + "iqr": 12330.533333333333, + "max": 49155.8, + "mean": 10243.953846153847, + "median": 4572.866666666667, + "min": 572.2666666666667, + "non_zero_mean": 10243.953846153847, + "non_zero_median": 4572.866666666667, + "percentile25": 3406.2, + "percentile75": 15736.733333333334, + "percentile90": 22762.066666666666, + "percentile99": 38403.19999999993, + "percentile999": 48080.54000000013, + "range": 48583.53333333333, + "samples": 91, + "stdev": 9735.024830798631, + "sum": 932199.7999999997 + }, + "pg_stat_database_tup_inserted": { + "iqr": 22.833333333333336, + "max": 91.86666666666666, + "mean": 14.291575091575092, + "median": 5.0, + "min": 0.0, + "non_zero_mean": 23.223809523809525, + "non_zero_median": 20.2, + "percentile25": 0.0, + "percentile75": 22.833333333333336, + "percentile90": 44.86666666666667, + "percentile99": 78.18666666666658, + "percentile999": 90.49866666666682, + "range": 91.86666666666666, + "samples": 91, + "stdev": 19.774208537739, + "sum": 1300.5333333333335 + }, + "pg_stat_database_tup_returned": { + "iqr": 17123.833333333336, + "max": 72595.06666666667, + "mean": 16274.65934065934, + "median": 8008.933333333333, + "min": 1490.9333333333334, + "non_zero_mean": 16274.65934065934, + "non_zero_median": 8008.933333333333, + "percentile25": 5883.633333333333, + "percentile75": 23007.466666666667, + "percentile90": 37081.86666666667, + "percentile99": 64201.90666666662, + "percentile999": 71755.75066666676, + "range": 71104.13333333333, + "samples": 91, + "stdev": 15027.606678644624, + "sum": 1480994.0000000005 + }, + "pg_stat_database_tup_updated": { + "iqr": 25.200000000000003, + "max": 62.266666666666666, + "mean": 14.712820512820512, + "median": 7.866666666666666, + "min": 0.0, + "non_zero_mean": 16.529218106995884, + "non_zero_median": 11.733333333333333, + "percentile25": 0.26666666666666666, + "percentile75": 25.46666666666667, + "percentile90": 40.4, + "percentile99": 54.766666666666616, + "percentile999": 61.51666666666676, + "range": 62.266666666666666, + "samples": 91, + "stdev": 16.535435028627003, + "sum": 1338.8666666666666 + }, + "pg_stat_database_xact_commit": { + "iqr": 225.63333333333333, + "max": 520.2, + "mean": 136.08205128205128, + "median": 89.06666666666666, + "min": 2.4, + "non_zero_mean": 136.08205128205128, + "non_zero_median": 89.06666666666666, + "percentile25": 9.266666666666667, + "percentile75": 234.9, + "percentile90": 331.0, + "percentile99": 482.33999999999975, + "percentile999": 516.4140000000004, + "range": 517.8000000000001, + "samples": 91, + "stdev": 141.02649189072508, + "sum": 12383.466666666674 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 91, + "stdev": 0.0, + "sum": 6.066666666666665 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.0029304029304029304, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.02666666666666515, + "percentile999": 0.24266666666666953, + "range": 0.26666666666666666, + "samples": 91, + "stdev": 0.027954262312584487, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 100.66666666666667, + "max": 187.0, + "mean": 54.22857142857143, + "median": 37.2, + "min": 0.26666666666666666, + "non_zero_mean": 54.22857142857143, + "non_zero_median": 37.2, + "percentile25": 4.2, + "percentile75": 104.86666666666667, + "percentile90": 127.06666666666666, + "percentile99": 174.8799999999999, + "percentile999": 185.78800000000015, + "range": 186.73333333333332, + "samples": 91, + "stdev": 54.033938254074634, + "sum": 4934.800000000002 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "60,61,62,69,66,72,67,63,70,64,74,75,68,83,71,79,77,81,80,78,76,84,82,88,86,87,90,89,94,95,97,93,96,91,98,99,101,100,102,105,107,109,108,106,111,110,112,115,113,114,116,118,117,120,104,121,122,124,103,123,133,134,132,131,135,139,127,136,128,129,141,140,144,147,146,149,152,145,150,143,126,154,153,155,157,158,160,159,162,163,166,165,168,156,161,164,169,167,170,171,172,175,173,176,174,177,178,180,179,184,181,190,182,192,183,186,189,187,188,193,191,185,195,196,194,197,200,198,199,202,206,208,204,203,201,205,211,209,207,210,212,214,219,215,218,213,216,220,223,222,221,224,225,226,229,231,230,232,238,217,233,237,239,236,240,234,245,241,243,249,227,247,228,242,244,255,246,235,250,252,254,251,257,256,253,260,248,258,259,263,262,265,261,264,267,266,268,270,269,271,275,274,279,273,277,272,276,282,285,286,289,281,284,288,283,292,287,280,294,295,291,293,290,296,278,297,298,303,299,300,301,304,306,307,302,310,312,311,317,314,315,305,308,316,323,318,319,322,321,326,328,309,325,313,324,320,329,334,327,333,331,337,330,340,336,343,332,339,342,348,346,338,352,341,345,335,351,355,356,357,347,358,344,349,350,354,363,361,353,360,365,370,366,369,359,368,364,362,367,371,373,372,374,375,384,376,379,377,382,378,381,383,380,386,385,388,392,389,387,397,390,391,395,393,396,394,399,401,400,404,402,403,398,408,405,412,409,406,407,413,417,410,418,415,419,411,416,420,421,414,425,422,423,424,427,428,426,430,429,433,435,438,443,441,439,442,432,444,436,431,437,440,449,447,446,450,445,455,452,458,456,451,454,453,460,463,457,466,459,468,461,470,462,467,465,434,471,469,448,464,472,478,474,486,480,473,482,481,475,477,483,490,476,488,479,487,484,492,485,491,494,489,498,499,496,493,500,502,495,497,503,504,505,501,506,509,507,514,512,511,510,508,515,518,516,517,513,519,523,522,524,520,525,526,528,531,527,529,532,521,533,534,539,540,536,537,538,543,546,547,535,541,548,542,550,554,555,552,545,551,553,556,560,549,558,530,557,559,561,562,563,565,544,569,567,564,568,566,570,571", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-18T23:25:13.977972Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-18T23:25:20.662261Z 2021-12-18T23:25:20.465123Z 2021-12-18T23:25:20.360503Z 2021-12-18T23:25:20.171235Z 2021-12-18T23:25:20.117868Z 2021-12-18T23:25:19.920209Z 2021-12-18T23:25:19.808278Z 2021-12-18T23:25:19.646012Z 2021-12-18T23:25:19.493694Z 2021-12-18T23:25:19.202874Z 2021-12-18T23:25:19.107307Z 2021-12-18T23:25:18.971832Z 2021-12-18T23:25:18.905126Z 2021-12-18T23:25:18.462622Z 2021-12-18T23:25:18.458026Z 2021-12-18T23:25:18.294310Z 2021-12-18T23:25:18.176537Z 2021-12-18T23:25:18.031910Z 2021-12-18T23:25:17.822568Z 2021-12-18T23:25:17.815669Z 2021-12-18T23:25:17.610384Z 2021-12-18T23:25:17.402967Z 2021-12-18T23:25:17.311085Z 2021-12-18T23:25:17.113776Z 2021-12-18T23:25:16.969388Z 2021-12-18T23:25:16.967049Z 2021-12-18T23:25:16.802510Z 2021-12-18T23:25:16.581467Z 2021-12-18T23:25:16.470227Z 2021-12-18T23:25:16.371270Z 2021-12-18T23:25:16.334151Z 2021-12-18T23:25:16.191860Z 2021-12-18T23:25:16.040104Z 2021-12-18T23:25:15.575739Z 2021-12-18T23:25:15.388525Z 2021-12-18T23:25:13.977972Z 2021-12-18T23:25:39.044435Z 2021-12-18T23:25:38.265640Z 2021-12-18T23:25:34.921423Z 2021-12-18T23:25:34.546310Z 2021-12-18T23:25:34.494491Z 2021-12-18T23:25:34.358484Z 2021-12-18T23:25:33.728960Z 2021-12-18T23:25:33.711278Z 2021-12-18T23:25:32.978361Z 2021-12-18T23:25:32.916902Z 2021-12-18T23:25:32.684302Z 2021-12-18T23:25:31.630168Z 2021-12-18T23:25:31.326144Z 2021-12-18T23:25:29.793818Z 2021-12-18T23:25:29.738636Z 2021-12-18T23:25:29.623824Z 2021-12-18T23:25:29.035609Z 2021-12-18T23:25:28.586678Z 2021-12-18T23:25:28.553774Z 2021-12-18T23:25:28.063996Z 2021-12-18T23:25:27.814720Z 2021-12-18T23:25:27.638032Z 2021-12-18T23:25:27.445491Z 2021-12-18T23:25:27.429813Z 2021-12-18T23:25:27.416928Z 2021-12-18T23:25:27.411168Z 2021-12-18T23:25:27.382503Z 2021-12-18T23:25:26.778938Z 2021-12-18T23:25:26.759330Z 2021-12-18T23:25:26.550953Z 2021-12-18T23:25:26.248456Z 2021-12-18T23:25:25.654376Z 2021-12-18T23:25:25.555933Z 2021-12-18T23:25:25.553727Z 2021-12-18T23:25:25.527174Z 2021-12-18T23:25:25.416543Z 2021-12-18T23:25:25.057387Z 2021-12-18T23:25:25.009713Z 2021-12-18T23:25:24.736590Z 2021-12-18T23:25:24.553783Z 2021-12-18T23:25:24.333803Z 2021-12-18T23:25:24.196694Z 2021-12-18T23:25:24.178328Z 2021-12-18T23:25:24.032639Z 2021-12-18T23:25:23.742464Z 2021-12-18T23:25:23.357043Z 2021-12-18T23:25:23.344764Z 2021-12-18T23:25:23.120495Z 2021-12-18T23:25:23.084975Z 2021-12-18T23:25:23.082300Z 2021-12-18T23:25:23.056128Z 2021-12-18T23:25:22.851097Z 2021-12-18T23:25:22.418650Z 2021-12-18T23:25:22.075197Z 2021-12-18T23:25:22.055108Z 2021-12-18T23:25:21.912815Z 2021-12-18T23:25:21.879210Z 2021-12-18T23:25:21.683047Z 2021-12-18T23:25:21.596720Z 2021-12-18T23:25:21.344537Z 2021-12-18T23:25:21.336611Z 2021-12-18T23:25:21.251567Z 2021-12-18T23:25:21.061085Z 2021-12-18T23:25:20.763843Z 2021-12-18T23:30:55.832803Z 2021-12-18T23:30:55.741082Z 2021-12-18T23:30:55.693917Z 2021-12-18T23:30:55.356866Z 2021-12-18T23:30:53.583256Z 2021-12-18T23:30:53.578083Z 2021-12-18T23:30:53.100771Z 2021-12-18T23:30:53.061686Z 2021-12-18T23:30:52.764593Z 2021-12-18T23:30:52.552857Z 2021-12-18T23:30:52.451905Z 2021-12-18T23:30:52.202664Z 2021-12-18T23:30:51.808994Z 2021-12-18T23:30:51.623912Z 2021-12-18T23:30:51.270674Z 2021-12-18T23:30:50.699903Z 2021-12-18T23:30:50.452835Z 2021-12-18T23:30:50.419815Z 2021-12-18T23:30:49.898424Z 2021-12-18T23:30:49.932949Z 2021-12-18T23:30:49.557056Z 2021-12-18T23:30:49.503509Z 2021-12-18T23:30:49.373930Z 2021-12-18T23:30:48.981594Z 2021-12-18T23:30:48.657731Z 2021-12-18T23:30:48.562041Z 2021-12-18T23:30:48.440974Z 2021-12-18T23:30:48.440200Z 2021-12-18T23:30:48.079771Z 2021-12-18T23:30:47.935216Z 2021-12-18T23:30:47.905870Z 2021-12-18T23:30:47.891271Z 2021-12-18T23:30:46.415177Z 2021-12-18T23:30:46.293549Z 2021-12-18T23:30:46.238260Z 2021-12-18T23:30:46.191043Z 2021-12-18T23:30:45.700617Z 2021-12-18T23:30:45.572712Z 2021-12-18T23:30:45.371774Z 2021-12-18T23:30:45.250357Z 2021-12-18T23:30:45.236913Z 2021-12-18T23:30:45.218489Z 2021-12-18T23:30:44.952650Z 2021-12-18T23:30:44.910769Z 2021-12-18T23:30:44.453748Z 2021-12-18T23:30:43.810924Z 2021-12-18T23:30:43.711519Z 2021-12-18T23:30:43.578485Z 2021-12-18T23:30:43.121805Z 2021-12-18T23:30:43.084980Z 2021-12-18T23:30:43.028100Z 2021-12-18T23:30:42.532501Z 2021-12-18T23:30:42.279279Z 2021-12-18T23:30:42.101116Z 2021-12-18T23:30:42.050426Z 2021-12-18T23:30:42.000044Z 2021-12-18T23:30:41.931204Z 2021-12-18T23:30:41.648659Z 2021-12-18T23:30:41.591470Z 2021-12-18T23:30:41.076026Z 2021-12-18T23:30:40.698141Z 2021-12-18T23:30:40.476113Z 2021-12-18T23:30:40.392417Z 2021-12-18T23:30:40.323309Z 2021-12-18T23:30:40.043752Z 2021-12-18T23:30:39.945740Z 2021-12-18T23:30:39.678351Z 2021-12-18T23:30:39.621489Z 2021-12-18T23:30:39.564185Z 2021-12-18T23:30:39.562670Z 2021-12-18T23:30:39.344413Z 2021-12-18T23:30:39.056535Z 2021-12-18T23:30:39.027934Z 2021-12-18T23:30:38.831011Z 2021-12-18T23:30:38.743042Z 2021-12-18T23:30:38.600693Z 2021-12-18T23:30:38.489136Z 2021-12-18T23:30:38.373038Z 2021-12-18T23:30:38.145183Z 2021-12-18T23:30:38.102629Z 2021-12-18T23:30:37.968422Z 2021-12-18T23:30:37.799615Z 2021-12-18T23:30:37.544794Z 2021-12-18T23:30:37.548136Z 2021-12-18T23:30:37.379448Z 2021-12-18T23:30:37.159198Z 2021-12-18T23:30:37.074059Z 2021-12-18T23:30:36.869322Z 2021-12-18T23:30:36.702827Z 2021-12-18T23:30:36.608690Z 2021-12-18T23:30:36.551678Z 2021-12-18T23:30:36.225646Z 2021-12-18T23:30:36.217101Z 2021-12-18T23:30:35.969519Z 2021-12-18T23:30:35.812581Z 2021-12-18T23:30:35.719481Z 2021-12-18T23:30:35.500529Z 2021-12-18T23:30:35.481463Z 2021-12-18T23:30:35.105562Z 2021-12-18T23:30:34.810182Z 2021-12-18T23:35:48.614103Z 2021-12-18T23:35:45.487236Z 2021-12-18T23:35:45.442472Z 2021-12-18T23:35:45.378415Z 2021-12-18T23:35:45.253714Z 2021-12-18T23:35:44.932952Z 2021-12-18T23:35:44.849408Z 2021-12-18T23:35:44.705733Z 2021-12-18T23:35:44.662673Z 2021-12-18T23:35:44.577215Z 2021-12-18T23:35:44.527174Z 2021-12-18T23:35:44.357172Z 2021-12-18T23:35:44.353116Z 2021-12-18T23:35:44.279092Z 2021-12-18T23:35:43.888191Z 2021-12-18T23:35:43.566600Z 2021-12-18T23:35:43.310449Z 2021-12-18T23:35:42.918462Z 2021-12-18T23:35:42.857433Z 2021-12-18T23:35:42.731810Z 2021-12-18T23:35:42.551992Z 2021-12-18T23:35:42.518091Z 2021-12-18T23:35:42.408460Z 2021-12-18T23:35:42.265435Z 2021-12-18T23:35:42.212633Z 2021-12-18T23:35:42.077210Z 2021-12-18T23:35:41.985639Z 2021-12-18T23:35:41.956258Z 2021-12-18T23:35:41.929570Z 2021-12-18T23:35:41.883356Z 2021-12-18T23:35:41.614678Z 2021-12-18T23:35:41.314404Z 2021-12-18T23:35:41.116499Z 2021-12-18T23:35:40.702437Z 2021-12-18T23:35:40.531361Z 2021-12-18T23:35:40.479462Z 2021-12-18T23:35:40.378142Z 2021-12-18T23:35:40.337682Z 2021-12-18T23:35:40.237495Z 2021-12-18T23:35:40.180816Z 2021-12-18T23:35:40.064831Z 2021-12-18T23:35:39.816296Z 2021-12-18T23:35:39.533182Z 2021-12-18T23:35:39.460369Z 2021-12-18T23:35:39.438560Z 2021-12-18T23:35:39.387496Z 2021-12-18T23:35:39.244644Z 2021-12-18T23:35:39.123887Z 2021-12-18T23:35:38.809522Z 2021-12-18T23:35:38.567550Z 2021-12-18T23:35:38.566001Z 2021-12-18T23:35:38.417839Z 2021-12-18T23:35:38.246549Z 2021-12-18T23:35:38.209203Z 2021-12-18T23:35:38.109559Z 2021-12-18T23:35:38.007003Z 2021-12-18T23:35:37.967582Z 2021-12-18T23:35:37.950516Z 2021-12-18T23:35:37.719741Z 2021-12-18T23:35:37.688032Z 2021-12-18T23:35:37.488151Z 2021-12-18T23:35:37.406401Z 2021-12-18T23:35:37.368039Z 2021-12-18T23:35:37.338588Z 2021-12-18T23:35:37.337489Z 2021-12-18T23:35:37.294444Z 2021-12-18T23:35:37.125356Z 2021-12-18T23:35:36.841531Z 2021-12-18T23:35:36.812930Z 2021-12-18T23:35:36.493573Z 2021-12-18T23:35:36.362940Z 2021-12-18T23:35:36.285495Z 2021-12-18T23:35:36.167278Z 2021-12-18T23:35:36.146677Z 2021-12-18T23:35:36.107504Z 2021-12-18T23:35:35.842640Z 2021-12-18T23:35:35.639361Z 2021-12-18T23:35:35.606654Z 2021-12-18T23:35:35.585528Z 2021-12-18T23:35:35.478900Z 2021-12-18T23:35:35.459546Z 2021-12-18T23:35:35.373180Z 2021-12-18T23:35:35.348339Z 2021-12-18T23:35:35.339323Z 2021-12-18T23:35:35.140175Z 2021-12-18T23:35:34.927722Z 2021-12-18T23:35:34.873812Z 2021-12-18T23:35:34.824802Z 2021-12-18T23:35:34.712897Z 2021-12-18T23:35:34.661143Z 2021-12-18T23:35:34.633297Z 2021-12-18T23:35:34.552656Z 2021-12-18T23:35:34.325620Z 2021-12-18T23:35:34.320365Z 2021-12-18T23:35:34.161787Z 2021-12-18T23:35:33.789431Z 2021-12-18T23:35:33.550040Z 2021-12-18T23:35:33.491252Z 2021-12-18T23:35:33.332493Z 2021-12-18T23:35:33.314800Z 2021-12-18T23:40:47.507551Z 2021-12-18T23:40:46.326287Z 2021-12-18T23:40:46.173099Z 2021-12-18T23:40:46.154215Z 2021-12-18T23:40:46.020207Z 2021-12-18T23:40:45.983216Z 2021-12-18T23:40:45.951719Z 2021-12-18T23:40:45.937826Z 2021-12-18T23:40:45.914385Z 2021-12-18T23:40:45.907710Z 2021-12-18T23:40:45.699768Z 2021-12-18T23:40:45.429963Z 2021-12-18T23:40:45.294162Z 2021-12-18T23:40:45.142760Z 2021-12-18T23:40:44.893339Z 2021-12-18T23:40:44.896639Z 2021-12-18T23:40:44.806642Z 2021-12-18T23:40:44.608659Z 2021-12-18T23:40:44.585685Z 2021-12-18T23:40:43.929511Z 2021-12-18T23:40:43.913815Z 2021-12-18T23:40:43.660952Z 2021-12-18T23:40:43.477321Z 2021-12-18T23:40:43.447800Z 2021-12-18T23:40:43.372454Z 2021-12-18T23:40:43.354806Z 2021-12-18T23:40:42.978711Z 2021-12-18T23:40:42.899590Z 2021-12-18T23:40:42.839179Z 2021-12-18T23:40:42.790760Z 2021-12-18T23:40:42.742547Z 2021-12-18T23:40:42.605626Z 2021-12-18T23:40:42.591103Z 2021-12-18T23:40:42.401076Z 2021-12-18T23:40:42.379206Z 2021-12-18T23:40:42.193386Z 2021-12-18T23:40:42.161728Z 2021-12-18T23:40:42.014954Z 2021-12-18T23:40:41.945630Z 2021-12-18T23:40:41.604375Z 2021-12-18T23:40:41.598564Z 2021-12-18T23:40:41.490496Z 2021-12-18T23:40:41.412903Z 2021-12-18T23:40:40.978571Z 2021-12-18T23:40:40.945347Z 2021-12-18T23:40:40.787232Z 2021-12-18T23:40:40.579583Z 2021-12-18T23:40:40.489788Z 2021-12-18T23:40:40.171692Z 2021-12-18T23:40:40.063435Z 2021-12-18T23:40:39.958344Z 2021-12-18T23:40:39.898184Z 2021-12-18T23:40:39.884207Z 2021-12-18T23:40:39.775611Z 2021-12-18T23:40:39.768684Z 2021-12-18T23:40:39.765726Z 2021-12-18T23:40:39.657608Z 2021-12-18T23:40:39.560274Z 2021-12-18T23:40:39.498435Z 2021-12-18T23:40:38.911216Z 2021-12-18T23:40:38.885658Z 2021-12-18T23:40:38.752025Z 2021-12-18T23:40:38.714961Z 2021-12-18T23:40:38.559820Z 2021-12-18T23:40:38.443050Z 2021-12-18T23:40:38.222260Z 2021-12-18T23:40:38.213098Z 2021-12-18T23:40:38.203031Z 2021-12-18T23:40:38.193446Z 2021-12-18T23:40:37.901676Z 2021-12-18T23:40:37.629201Z 2021-12-18T23:40:37.596916Z 2021-12-18T23:40:37.490703Z 2021-12-18T23:40:37.428468Z 2021-12-18T23:40:37.313576Z 2021-12-18T23:40:37.222624Z 2021-12-18T23:40:36.923736Z 2021-12-18T23:40:36.930260Z 2021-12-18T23:40:36.814581Z 2021-12-18T23:40:36.796630Z 2021-12-18T23:40:36.624275Z 2021-12-18T23:40:36.593892Z 2021-12-18T23:40:36.341160Z 2021-12-18T23:40:36.283859Z 2021-12-18T23:40:36.324685Z 2021-12-18T23:40:36.138457Z 2021-12-18T23:40:36.008597Z 2021-12-18T23:40:35.856920Z 2021-12-18T23:40:35.743111Z 2021-12-18T23:40:35.704608Z 2021-12-18T23:40:35.673088Z 2021-12-18T23:40:35.563113Z 2021-12-18T23:40:35.514915Z 2021-12-18T23:40:35.453891Z 2021-12-18T23:40:35.416338Z 2021-12-18T23:40:35.355643Z 2021-12-18T23:40:35.051093Z 2021-12-18T23:40:34.908899Z 2021-12-18T23:40:34.393339Z 2021-12-18T23:40:33.994654Z 2021-12-18T23:45:54.759963Z 2021-12-18T23:45:54.586733Z 2021-12-18T23:45:54.347805Z 2021-12-18T23:45:54.245680Z 2021-12-18T23:45:54.212395Z 2021-12-18T23:45:54.099537Z 2021-12-18T23:45:54.045893Z 2021-12-18T23:45:54.028050Z 2021-12-18T23:45:53.887378Z 2021-12-18T23:45:53.508174Z 2021-12-18T23:45:52.634933Z 2021-12-18T23:45:52.598322Z 2021-12-18T23:45:52.512147Z 2021-12-18T23:45:52.464404Z 2021-12-18T23:45:52.025986Z 2021-12-18T23:45:51.993238Z 2021-12-18T23:45:51.948509Z 2021-12-18T23:45:51.935673Z 2021-12-18T23:45:51.796371Z 2021-12-18T23:45:51.780456Z 2021-12-18T23:45:51.709433Z 2021-12-18T23:45:51.636128Z 2021-12-18T23:45:51.592228Z 2021-12-18T23:45:51.583967Z 2021-12-18T23:45:51.432040Z 2021-12-18T23:45:51.154619Z 2021-12-18T23:45:50.999240Z 2021-12-18T23:45:50.855505Z 2021-12-18T23:45:50.787295Z 2021-12-18T23:45:50.686326Z 2021-12-18T23:45:50.668975Z 2021-12-18T23:45:50.619112Z 2021-12-18T23:45:50.448868Z 2021-12-18T23:45:50.360322Z 2021-12-18T23:45:50.374439Z 2021-12-18T23:45:50.035579Z 2021-12-18T23:45:49.998803Z 2021-12-18T23:45:49.903568Z 2021-12-18T23:45:49.492338Z 2021-12-18T23:45:49.185223Z 2021-12-18T23:45:49.152461Z 2021-12-18T23:45:48.968351Z 2021-12-18T23:45:48.863862Z 2021-12-18T23:45:48.619836Z 2021-12-18T23:45:48.487301Z 2021-12-18T23:45:48.372604Z 2021-12-18T23:45:48.340657Z 2021-12-18T23:45:48.159128Z 2021-12-18T23:45:48.137829Z 2021-12-18T23:45:48.119618Z 2021-12-18T23:45:48.020761Z 2021-12-18T23:45:47.953343Z 2021-12-18T23:45:47.796232Z 2021-12-18T23:45:47.604755Z 2021-12-18T23:45:47.499640Z 2021-12-18T23:45:47.478161Z 2021-12-18T23:45:47.420588Z 2021-12-18T23:45:47.382627Z 2021-12-18T23:45:47.392465Z 2021-12-18T23:45:47.346712Z 2021-12-18T23:45:47.285692Z 2021-12-18T23:45:46.836071Z 2021-12-18T23:45:46.607863Z 2021-12-18T23:45:46.490305Z 2021-12-18T23:45:46.447322Z 2021-12-18T23:45:46.333887Z 2021-12-18T23:45:46.200123Z 2021-12-18T23:45:45.945787Z 2021-12-18T23:45:45.819003Z 2021-12-18T23:45:45.723440Z 2021-12-18T23:45:45.614681Z 2021-12-18T23:45:45.584226Z 2021-12-18T23:45:45.508257Z 2021-12-18T23:45:45.473608Z 2021-12-18T23:45:45.447312Z 2021-12-18T23:45:45.273349Z 2021-12-18T23:45:45.190135Z 2021-12-18T23:45:45.178222Z 2021-12-18T23:45:45.178578Z 2021-12-18T23:45:44.863417Z 2021-12-18T23:45:44.817121Z 2021-12-18T23:45:44.798571Z 2021-12-18T23:45:44.742570Z 2021-12-18T23:45:44.676469Z 2021-12-18T23:45:44.681517Z 2021-12-18T23:45:44.668514Z 2021-12-18T23:45:44.462402Z 2021-12-18T23:45:44.302449Z 2021-12-18T23:45:44.259603Z 2021-12-18T23:45:43.891001Z 2021-12-18T23:45:43.801339Z 2021-12-18T23:45:43.712685Z 2021-12-18T23:45:43.701430Z 2021-12-18T23:45:43.516732Z 2021-12-18T23:45:43.373660Z 2021-12-18T23:45:43.326082Z 2021-12-18T23:45:43.264147Z 2021-12-18T23:45:42.931569Z 2021-12-18T23:45:42.865337Z 2021-12-18T23:45:42.559317Z", + "created_diff": 17.420386, + "ended": "2021-12-18T23:47:51.193205Z", + "error_job_ids": " []", + "event_first": "2021-12-18T23:26:46.838527Z 2021-12-18T23:26:17.187758Z 2021-12-18T23:26:49.130672Z 2021-12-18T23:26:09.567204Z 2021-12-18T23:26:47.826368Z 2021-12-18T23:25:57.955034Z 2021-12-18T23:26:30.184735Z 2021-12-18T23:25:56.505612Z 2021-12-18T23:26:38.200196Z 2021-12-18T23:25:51.372915Z 2021-12-18T23:26:26.252415Z 2021-12-18T23:26:31.953621Z 2021-12-18T23:28:10.674630Z 2021-12-18T23:25:50.171620Z 2021-12-18T23:26:21.026124Z 2021-12-18T23:25:54.006971Z 2021-12-18T23:27:34.539564Z 2021-12-18T23:25:51.727579Z 2021-12-18T23:27:15.727543Z 2021-12-18T23:25:52.685491Z 2021-12-18T23:27:51.548713Z 2021-12-18T23:25:47.005517Z 2021-12-18T23:27:43.703052Z 2021-12-18T23:25:45.439569Z 2021-12-18T23:27:06.968490Z 2021-12-18T23:25:42.916536Z 2021-12-18T23:28:01.583027Z 2021-12-18T23:25:39.231414Z 2021-12-18T23:25:42.861799Z 2021-12-18T23:25:31.484398Z 2021-12-18T23:26:57.511672Z 2021-12-18T23:25:32.042670Z 2021-12-18T23:27:25.867681Z 2021-12-18T23:25:28.854405Z 2021-12-18T23:25:33.738030Z 2021-12-18T23:25:21.211073Z 2021-12-18T23:27:08.903354Z 2021-12-18T23:26:50.948541Z 2021-12-18T23:27:08.903354Z 2021-12-18T23:26:51.737679Z 2021-12-18T23:27:10.574477Z 2021-12-18T23:26:54.909092Z 2021-12-18T23:27:09.875298Z 2021-12-18T23:26:53.456096Z 2021-12-18T23:27:06.185308Z 2021-12-18T23:26:52.856879Z 2021-12-18T23:27:07.899501Z 2021-12-18T23:26:48.637486Z 2021-12-18T23:27:09.160285Z 2021-12-18T23:26:53.133938Z 2021-12-18T23:27:05.994032Z 2021-12-18T23:26:49.924056Z 2021-12-18T23:27:07.462474Z 2021-12-18T23:26:45.352517Z 2021-12-18T23:27:00.386327Z 2021-12-18T23:26:51.737679Z 2021-12-18T23:27:00.503417Z 2021-12-18T23:26:50.984748Z 2021-12-18T23:27:11.834863Z 2021-12-18T23:27:10.904080Z 2021-12-18T23:27:05.680468Z 2021-12-18T23:27:11.231691Z 2021-12-18T23:27:09.386263Z 2021-12-18T23:26:52.724658Z 2021-12-18T23:26:45.109735Z 2021-12-18T23:27:04.149995Z 2021-12-18T23:26:45.654015Z 2021-12-18T23:26:57.854889Z 2021-12-18T23:26:42.321579Z 2021-12-18T23:26:58.698758Z 2021-12-18T23:26:44.095622Z 2021-12-18T23:27:04.149995Z 2021-12-18T23:26:36.225257Z 2021-12-18T23:26:57.646365Z 2021-12-18T23:26:42.331998Z 2021-12-18T23:26:59.426294Z 2021-12-18T23:26:50.954677Z 2021-12-18T23:26:34.296999Z 2021-12-18T23:26:53.472481Z 2021-12-18T23:26:38.086767Z 2021-12-18T23:26:58.930221Z 2021-12-18T23:26:42.565634Z 2021-12-18T23:26:52.176469Z 2021-12-18T23:26:37.828771Z 2021-12-18T23:26:51.453169Z 2021-12-18T23:26:34.620745Z 2021-12-18T23:26:52.251639Z 2021-12-18T23:26:38.547667Z 2021-12-18T23:26:46.122334Z 2021-12-18T23:26:25.929353Z 2021-12-18T23:26:47.701858Z 2021-12-18T23:26:25.420760Z 2021-12-18T23:26:38.032661Z 2021-12-18T23:26:29.435626Z 2021-12-18T23:26:46.585096Z 2021-12-18T23:26:28.684213Z 2021-12-18T23:26:29.713262Z 2021-12-18T23:26:26.181044Z 2021-12-18T23:26:58.747377Z 2021-12-18T23:26:26.940615Z 2021-12-18T23:32:26.788707Z 2021-12-18T23:32:40.533333Z 2021-12-18T23:32:25.651577Z 2021-12-18T23:32:42.419424Z 2021-12-18T23:32:19.159208Z 2021-12-18T23:32:37.970518Z 2021-12-18T23:32:26.556181Z 2021-12-18T23:32:43.333623Z 2021-12-18T23:32:20.683967Z 2021-12-18T23:32:40.961129Z 2021-12-18T23:32:25.896731Z 2021-12-18T23:32:40.049792Z 2021-12-18T23:32:13.900470Z 2021-12-18T23:32:44.163535Z 2021-12-18T23:32:25.910685Z 2021-12-18T23:32:31.864984Z 2021-12-18T23:32:24.745650Z 2021-12-18T23:32:42.024418Z 2021-12-18T23:32:38.500371Z 2021-12-18T23:32:24.745650Z 2021-12-18T23:32:17.693143Z 2021-12-18T23:32:42.931869Z 2021-12-18T23:32:20.889217Z 2021-12-18T23:32:34.684671Z 2021-12-18T23:32:17.666043Z 2021-12-18T23:32:41.783710Z 2021-12-18T23:32:16.411109Z 2021-12-18T23:32:29.575331Z 2021-12-18T23:32:21.156733Z 2021-12-18T23:32:44.163535Z 2021-12-18T23:32:17.693143Z 2021-12-18T23:32:38.500371Z 2021-12-18T23:32:18.272204Z 2021-12-18T23:32:35.006444Z 2021-12-18T23:32:14.877104Z 2021-12-18T23:32:32.865129Z 2021-12-18T23:32:13.256905Z 2021-12-18T23:32:26.043590Z 2021-12-18T23:32:15.631434Z 2021-12-18T23:32:36.451666Z 2021-12-18T23:32:15.631434Z 2021-12-18T23:32:39.452367Z 2021-12-18T23:32:18.272204Z 2021-12-18T23:32:30.672094Z 2021-12-18T23:32:12.940996Z 2021-12-18T23:32:28.014397Z 2021-12-18T23:32:09.137177Z 2021-12-18T23:32:30.335376Z 2021-12-18T23:32:12.247817Z 2021-12-18T23:32:23.009990Z 2021-12-18T23:32:10.168952Z 2021-12-18T23:32:27.701283Z 2021-12-18T23:32:16.263110Z 2021-12-18T23:31:51.744922Z 2021-12-18T23:32:25.814469Z 2021-12-18T23:32:13.701105Z 2021-12-18T23:31:52.704798Z 2021-12-18T23:32:18.164633Z 2021-12-18T23:31:57.114894Z 2021-12-18T23:32:09.333860Z 2021-12-18T23:32:01.502177Z 2021-12-18T23:32:05.410843Z 2021-12-18T23:31:48.424998Z 2021-12-18T23:31:57.835441Z 2021-12-18T23:31:55.390767Z 2021-12-18T23:32:06.939807Z 2021-12-18T23:31:47.487810Z 2021-12-18T23:32:11.388123Z 2021-12-18T23:31:38.351393Z 2021-12-18T23:32:01.634070Z 2021-12-18T23:31:41.129638Z 2021-12-18T23:32:06.286717Z 2021-12-18T23:31:27.490589Z 2021-12-18T23:31:58.500041Z 2021-12-18T23:31:30.564346Z 2021-12-18T23:31:37.584054Z 2021-12-18T23:31:51.014818Z 2021-12-18T23:31:29.980636Z 2021-12-18T23:31:46.318780Z 2021-12-18T23:31:39.363264Z 2021-12-18T23:31:34.365718Z 2021-12-18T23:31:30.213974Z 2021-12-18T23:31:27.797561Z 2021-12-18T23:31:21.803908Z 2021-12-18T23:31:08.779742Z 2021-12-18T23:31:20.685698Z 2021-12-18T23:31:04.841588Z 2021-12-18T23:31:05.411827Z 2021-12-18T23:30:58.736351Z 2021-12-18T23:31:00.093602Z 2021-12-18T23:30:58.996560Z 2021-12-18T23:31:01.099938Z 2021-12-18T23:30:59.504549Z 2021-12-18T23:30:51.809772Z 2021-12-18T23:30:52.288543Z 2021-12-18T23:30:50.799768Z 2021-12-18T23:30:54.245607Z 2021-12-18T23:30:45.761926Z 2021-12-18T23:30:47.581182Z 2021-12-18T23:37:34.649418Z 2021-12-18T23:37:40.032646Z 2021-12-18T23:37:37.093366Z 2021-12-18T23:37:35.932659Z 2021-12-18T23:37:37.046246Z 2021-12-18T23:37:34.928024Z 2021-12-18T23:37:34.401579Z 2021-12-18T23:37:44.897016Z 2021-12-18T23:37:36.041183Z 2021-12-18T23:37:39.497339Z 2021-12-18T23:37:32.256837Z 2021-12-18T23:37:42.948437Z 2021-12-18T23:37:29.737033Z 2021-12-18T23:37:36.945876Z 2021-12-18T23:37:34.141677Z 2021-12-18T23:37:44.147054Z 2021-12-18T23:37:25.001537Z 2021-12-18T23:37:34.420880Z 2021-12-18T23:37:23.009205Z 2021-12-18T23:37:36.192709Z 2021-12-18T23:37:31.152885Z 2021-12-18T23:37:39.243156Z 2021-12-18T23:37:31.017400Z 2021-12-18T23:37:39.491507Z 2021-12-18T23:37:27.959037Z 2021-12-18T23:37:32.446554Z 2021-12-18T23:37:34.010607Z 2021-12-18T23:37:41.291598Z 2021-12-18T23:37:21.497598Z 2021-12-18T23:37:33.651796Z 2021-12-18T23:37:28.508001Z 2021-12-18T23:37:30.635597Z 2021-12-18T23:37:25.532252Z 2021-12-18T23:37:32.053533Z 2021-12-18T23:37:27.889757Z 2021-12-18T23:37:33.061512Z 2021-12-18T23:37:28.219366Z 2021-12-18T23:37:30.881961Z 2021-12-18T23:37:14.266528Z 2021-12-18T23:37:25.254036Z 2021-12-18T23:37:21.470236Z 2021-12-18T23:37:28.020749Z 2021-12-18T23:37:25.269368Z 2021-12-18T23:37:30.630185Z 2021-12-18T23:37:23.993521Z 2021-12-18T23:37:27.591811Z 2021-12-18T23:37:08.166294Z 2021-12-18T23:37:28.849675Z 2021-12-18T23:37:21.219575Z 2021-12-18T23:37:26.591334Z 2021-12-18T23:37:13.752175Z 2021-12-18T23:37:29.631133Z 2021-12-18T23:37:19.051029Z 2021-12-18T23:37:29.253473Z 2021-12-18T23:37:09.274531Z 2021-12-18T23:37:15.762758Z 2021-12-18T23:37:23.866394Z 2021-12-18T23:37:22.222316Z 2021-12-18T23:36:57.798732Z 2021-12-18T23:37:19.409313Z 2021-12-18T23:37:10.454353Z 2021-12-18T23:37:11.964709Z 2021-12-18T23:37:07.394544Z 2021-12-18T23:37:06.884715Z 2021-12-18T23:37:13.517666Z 2021-12-18T23:37:18.270539Z 2021-12-18T23:36:55.504619Z 2021-12-18T23:37:10.940798Z 2021-12-18T23:37:04.391868Z 2021-12-18T23:37:05.010109Z 2021-12-18T23:36:55.998572Z 2021-12-18T23:37:06.114763Z 2021-12-18T23:36:53.227234Z 2021-12-18T23:37:06.880808Z 2021-12-18T23:36:51.803611Z 2021-12-18T23:37:00.552906Z 2021-12-18T23:36:21.712128Z 2021-12-18T23:36:17.219423Z 2021-12-18T23:36:15.671975Z 2021-12-18T23:36:13.985365Z 2021-12-18T23:36:17.673860Z 2021-12-18T23:36:09.652016Z 2021-12-18T23:36:23.185856Z 2021-12-18T23:36:10.893486Z 2021-12-18T23:36:06.922515Z 2021-12-18T23:36:14.415545Z 2021-12-18T23:36:11.037191Z 2021-12-18T23:36:09.363890Z 2021-12-18T23:36:08.955693Z 2021-12-18T23:36:06.478038Z 2021-12-18T23:36:02.131692Z 2021-12-18T23:36:02.597431Z 2021-12-18T23:36:00.665895Z 2021-12-18T23:36:04.376379Z 2021-12-18T23:35:58.727581Z 2021-12-18T23:36:01.871710Z 2021-12-18T23:35:47.439541Z 2021-12-18T23:35:49.132708Z 2021-12-18T23:35:46.610375Z 2021-12-18T23:35:46.717557Z 2021-12-18T23:42:41.598863Z 2021-12-18T23:42:40.086382Z 2021-12-18T23:42:37.586878Z 2021-12-18T23:42:47.018805Z 2021-12-18T23:42:34.488722Z 2021-12-18T23:42:35.128524Z 2021-12-18T23:42:33.148764Z 2021-12-18T23:42:43.657691Z 2021-12-18T23:42:37.586878Z 2021-12-18T23:42:46.449283Z 2021-12-18T23:42:38.123871Z 2021-12-18T23:42:45.178497Z 2021-12-18T23:42:40.881503Z 2021-12-18T23:42:34.358047Z 2021-12-18T23:42:43.656532Z 2021-12-18T23:42:31.236810Z 2021-12-18T23:42:40.222429Z 2021-12-18T23:42:40.340455Z 2021-12-18T23:42:35.552018Z 2021-12-18T23:42:39.709484Z 2021-12-18T23:42:37.664799Z 2021-12-18T23:42:37.239011Z 2021-12-18T23:42:33.282338Z 2021-12-18T23:42:41.110629Z 2021-12-18T23:42:35.236755Z 2021-12-18T23:42:41.305360Z 2021-12-18T23:42:33.533197Z 2021-12-18T23:42:38.370360Z 2021-12-18T23:42:37.664799Z 2021-12-18T23:42:39.833427Z 2021-12-18T23:42:33.401012Z 2021-12-18T23:42:30.080081Z 2021-12-18T23:42:28.472810Z 2021-12-18T23:42:40.340455Z 2021-12-18T23:42:37.586878Z 2021-12-18T23:42:33.608951Z 2021-12-18T23:42:30.984312Z 2021-12-18T23:42:33.126452Z 2021-12-18T23:42:27.576786Z 2021-12-18T23:42:34.619522Z 2021-12-18T23:42:29.467884Z 2021-12-18T23:42:30.840887Z 2021-12-18T23:42:29.462882Z 2021-12-18T23:42:27.312510Z 2021-12-18T23:42:29.464915Z 2021-12-18T23:42:33.056759Z 2021-12-18T23:42:23.913728Z 2021-12-18T23:42:14.134108Z 2021-12-18T23:42:11.800207Z 2021-12-18T23:42:21.039639Z 2021-12-18T23:42:20.006306Z 2021-12-18T23:42:25.237678Z 2021-12-18T23:42:20.931027Z 2021-12-18T23:42:20.463510Z 2021-12-18T23:42:14.898972Z 2021-12-18T23:42:18.375383Z 2021-12-18T23:42:19.492912Z 2021-12-18T23:42:22.623484Z 2021-12-18T23:42:14.586934Z 2021-12-18T23:42:27.810802Z 2021-12-18T23:42:10.695791Z 2021-12-18T23:42:15.133277Z 2021-12-18T23:42:05.901958Z 2021-12-18T23:42:19.266289Z 2021-12-18T23:42:14.898972Z 2021-12-18T23:42:21.060788Z 2021-12-18T23:42:09.442773Z 2021-12-18T23:42:16.604731Z 2021-12-18T23:42:10.107590Z 2021-12-18T23:42:10.818633Z 2021-12-18T23:42:10.954718Z 2021-12-18T23:42:16.288128Z 2021-12-18T23:41:59.535101Z 2021-12-18T23:42:12.513596Z 2021-12-18T23:41:39.333532Z 2021-12-18T23:42:05.138534Z 2021-12-18T23:41:51.017436Z 2021-12-18T23:41:43.338555Z 2021-12-18T23:41:30.401341Z 2021-12-18T23:41:29.550774Z 2021-12-18T23:41:20.082845Z 2021-12-18T23:41:40.850957Z 2021-12-18T23:41:28.958896Z 2021-12-18T23:41:15.695885Z 2021-12-18T23:41:41.526105Z 2021-12-18T23:41:25.428733Z 2021-12-18T23:41:15.911283Z 2021-12-18T23:41:12.877215Z 2021-12-18T23:41:15.228156Z 2021-12-18T23:41:19.607494Z 2021-12-18T23:41:08.041002Z 2021-12-18T23:41:14.714104Z 2021-12-18T23:41:03.693132Z 2021-12-18T23:41:15.083826Z 2021-12-18T23:41:02.490291Z 2021-12-18T23:41:01.154490Z 2021-12-18T23:40:50.416410Z 2021-12-18T23:40:51.027738Z 2021-12-18T23:40:47.619278Z 2021-12-18T23:40:44.163039Z 2021-12-18T23:47:46.998339Z 2021-12-18T23:47:43.489514Z 2021-12-18T23:47:43.459603Z 2021-12-18T23:47:46.166528Z 2021-12-18T23:47:41.910755Z 2021-12-18T23:47:43.740149Z 2021-12-18T23:47:47.546748Z 2021-12-18T23:47:46.575711Z 2021-12-18T23:47:46.530503Z 2021-12-18T23:47:45.221737Z 2021-12-18T23:47:43.424379Z 2021-12-18T23:47:46.166528Z 2021-12-18T23:47:46.357736Z 2021-12-18T23:47:43.740149Z 2021-12-18T23:47:38.865477Z 2021-12-18T23:47:43.840086Z 2021-12-18T23:47:42.718597Z 2021-12-18T23:47:47.503275Z 2021-12-18T23:47:37.837374Z 2021-12-18T23:47:44.092685Z 2021-12-18T23:47:36.730650Z 2021-12-18T23:47:41.588986Z 2021-12-18T23:47:40.213045Z 2021-12-18T23:47:35.352821Z 2021-12-18T23:47:37.483337Z 2021-12-18T23:47:37.779442Z 2021-12-18T23:47:34.433431Z 2021-12-18T23:47:37.075071Z 2021-12-18T23:47:35.486993Z 2021-12-18T23:47:31.181763Z 2021-12-18T23:47:27.080529Z 2021-12-18T23:47:37.380466Z 2021-12-18T23:47:38.197473Z 2021-12-18T23:47:35.217474Z 2021-12-18T23:47:34.088983Z 2021-12-18T23:47:39.791727Z 2021-12-18T23:47:24.399005Z 2021-12-18T23:47:36.360208Z 2021-12-18T23:47:29.304335Z 2021-12-18T23:47:32.704903Z 2021-12-18T23:47:32.379486Z 2021-12-18T23:47:25.542659Z 2021-12-18T23:47:23.781468Z 2021-12-18T23:47:29.870533Z 2021-12-18T23:47:22.241374Z 2021-12-18T23:47:18.024497Z 2021-12-18T23:47:28.659980Z 2021-12-18T23:47:25.435696Z 2021-12-18T23:47:23.032972Z 2021-12-18T23:47:22.536883Z 2021-12-18T23:47:19.947894Z 2021-12-18T23:47:21.238825Z 2021-12-18T23:47:24.176073Z 2021-12-18T23:46:49.969628Z 2021-12-18T23:46:50.524841Z 2021-12-18T23:46:52.523891Z 2021-12-18T23:46:42.558147Z 2021-12-18T23:46:46.552012Z 2021-12-18T23:46:48.738900Z 2021-12-18T23:46:44.422195Z 2021-12-18T23:46:47.968319Z 2021-12-18T23:46:48.827558Z 2021-12-18T23:46:45.496777Z 2021-12-18T23:46:45.117367Z 2021-12-18T23:46:42.220882Z 2021-12-18T23:46:47.517846Z 2021-12-18T23:46:41.461773Z 2021-12-18T23:46:46.115638Z 2021-12-18T23:46:44.989005Z 2021-12-18T23:46:41.664480Z 2021-12-18T23:46:38.962582Z 2021-12-18T23:46:38.904830Z 2021-12-18T23:46:38.709016Z 2021-12-18T23:46:43.847549Z 2021-12-18T23:46:36.148627Z 2021-12-18T23:46:33.422899Z 2021-12-18T23:46:38.903632Z 2021-12-18T23:46:35.961032Z 2021-12-18T23:46:37.096527Z 2021-12-18T23:46:36.315455Z 2021-12-18T23:46:34.379094Z 2021-12-18T23:46:37.099106Z 2021-12-18T23:46:26.302042Z 2021-12-18T23:46:27.146671Z 2021-12-18T23:46:24.015772Z 2021-12-18T23:46:32.352073Z 2021-12-18T23:46:31.849859Z 2021-12-18T23:46:34.407789Z 2021-12-18T23:46:24.264783Z 2021-12-18T23:46:30.694888Z 2021-12-18T23:46:26.059435Z 2021-12-18T23:46:31.650032Z 2021-12-18T23:46:23.782966Z 2021-12-18T23:46:26.298106Z 2021-12-18T23:46:21.122424Z 2021-12-18T23:45:59.448997Z 2021-12-18T23:46:22.189417Z 2021-12-18T23:46:00.431776Z 2021-12-18T23:45:56.860131Z 2021-12-18T23:45:54.395830Z", + "event_last": "2021-12-18T23:26:52.898604Z 2021-12-18T23:26:21.106818Z 2021-12-18T23:26:54.068220Z 2021-12-18T23:26:14.074540Z 2021-12-18T23:26:53.525923Z 2021-12-18T23:26:01.112947Z 2021-12-18T23:26:35.553215Z 2021-12-18T23:25:59.740539Z 2021-12-18T23:26:41.945127Z 2021-12-18T23:25:53.103323Z 2021-12-18T23:26:30.691717Z 2021-12-18T23:26:37.731688Z 2021-12-18T23:28:13.023300Z 2021-12-18T23:25:52.768687Z 2021-12-18T23:26:25.058065Z 2021-12-18T23:25:57.229746Z 2021-12-18T23:27:35.004989Z 2021-12-18T23:25:54.039488Z 2021-12-18T23:27:16.159861Z 2021-12-18T23:25:55.939152Z 2021-12-18T23:27:52.159584Z 2021-12-18T23:25:48.983258Z 2021-12-18T23:27:45.410171Z 2021-12-18T23:25:47.644842Z 2021-12-18T23:27:07.689481Z 2021-12-18T23:25:44.494673Z 2021-12-18T23:28:02.007917Z 2021-12-18T23:25:41.067264Z 2021-12-18T23:25:43.627012Z 2021-12-18T23:25:34.142068Z 2021-12-18T23:26:58.108053Z 2021-12-18T23:25:34.280365Z 2021-12-18T23:27:27.937260Z 2021-12-18T23:25:30.426165Z 2021-12-18T23:25:33.926177Z 2021-12-18T23:25:22.208304Z 2021-12-18T23:27:11.045711Z 2021-12-18T23:26:53.166635Z 2021-12-18T23:27:10.969485Z 2021-12-18T23:26:53.531645Z 2021-12-18T23:27:13.418293Z 2021-12-18T23:26:55.951158Z 2021-12-18T23:27:12.256217Z 2021-12-18T23:26:55.291262Z 2021-12-18T23:27:08.476901Z 2021-12-18T23:26:54.855336Z 2021-12-18T23:27:11.056422Z 2021-12-18T23:26:51.117100Z 2021-12-18T23:27:11.979292Z 2021-12-18T23:26:54.554470Z 2021-12-18T23:27:08.625295Z 2021-12-18T23:26:51.825472Z 2021-12-18T23:27:10.299881Z 2021-12-18T23:26:49.144675Z 2021-12-18T23:27:04.854210Z 2021-12-18T23:26:53.825500Z 2021-12-18T23:27:05.917657Z 2021-12-18T23:26:53.166635Z 2021-12-18T23:27:13.304190Z 2021-12-18T23:27:12.706520Z 2021-12-18T23:27:08.097373Z 2021-12-18T23:27:13.216247Z 2021-12-18T23:27:12.599327Z 2021-12-18T23:26:57.078656Z 2021-12-18T23:26:48.654025Z 2021-12-18T23:27:07.033937Z 2021-12-18T23:26:48.809495Z 2021-12-18T23:27:02.146761Z 2021-12-18T23:26:45.776437Z 2021-12-18T23:27:03.184182Z 2021-12-18T23:26:47.294457Z 2021-12-18T23:27:07.048069Z 2021-12-18T23:26:39.877439Z 2021-12-18T23:27:01.644066Z 2021-12-18T23:26:44.612745Z 2021-12-18T23:27:03.864463Z 2021-12-18T23:26:55.356535Z 2021-12-18T23:26:38.027378Z 2021-12-18T23:26:56.080160Z 2021-12-18T23:26:41.489105Z 2021-12-18T23:27:04.770156Z 2021-12-18T23:26:45.489446Z 2021-12-18T23:26:57.525428Z 2021-12-18T23:26:41.487774Z 2021-12-18T23:26:57.361159Z 2021-12-18T23:26:38.399288Z 2021-12-18T23:26:57.205372Z 2021-12-18T23:26:41.380487Z 2021-12-18T23:26:51.924183Z 2021-12-18T23:26:29.017492Z 2021-12-18T23:26:53.223953Z 2021-12-18T23:26:28.783465Z 2021-12-18T23:26:42.574993Z 2021-12-18T23:26:32.742514Z 2021-12-18T23:26:50.758205Z 2021-12-18T23:26:31.790781Z 2021-12-18T23:26:35.975151Z 2021-12-18T23:26:30.796468Z 2021-12-18T23:27:02.232246Z 2021-12-18T23:26:30.722216Z 2021-12-18T23:32:31.052532Z 2021-12-18T23:32:43.749948Z 2021-12-18T23:32:27.393597Z 2021-12-18T23:32:45.489443Z 2021-12-18T23:32:23.198573Z 2021-12-18T23:32:41.702250Z 2021-12-18T23:32:28.299928Z 2021-12-18T23:32:46.097800Z 2021-12-18T23:32:24.318369Z 2021-12-18T23:32:44.251837Z 2021-12-18T23:32:28.299928Z 2021-12-18T23:32:43.934313Z 2021-12-18T23:32:17.153813Z 2021-12-18T23:32:49.012777Z 2021-12-18T23:32:27.863613Z 2021-12-18T23:32:36.532240Z 2021-12-18T23:32:26.769571Z 2021-12-18T23:32:46.200055Z 2021-12-18T23:32:42.322492Z 2021-12-18T23:32:27.146101Z 2021-12-18T23:32:20.963554Z 2021-12-18T23:32:46.043810Z 2021-12-18T23:32:24.567381Z 2021-12-18T23:32:38.825189Z 2021-12-18T23:32:22.246456Z 2021-12-18T23:32:44.945146Z 2021-12-18T23:32:19.928553Z 2021-12-18T23:32:32.863381Z 2021-12-18T23:32:25.193784Z 2021-12-18T23:32:46.714258Z 2021-12-18T23:32:23.098414Z 2021-12-18T23:32:41.843779Z 2021-12-18T23:32:21.530965Z 2021-12-18T23:32:38.850218Z 2021-12-18T23:32:18.127535Z 2021-12-18T23:32:36.638239Z 2021-12-18T23:32:16.542120Z 2021-12-18T23:32:29.580230Z 2021-12-18T23:32:19.767501Z 2021-12-18T23:32:40.118479Z 2021-12-18T23:32:19.254206Z 2021-12-18T23:32:43.057441Z 2021-12-18T23:32:22.514974Z 2021-12-18T23:32:35.191174Z 2021-12-18T23:32:16.535492Z 2021-12-18T23:32:32.933197Z 2021-12-18T23:32:12.652806Z 2021-12-18T23:32:34.698005Z 2021-12-18T23:32:15.398542Z 2021-12-18T23:32:27.134797Z 2021-12-18T23:32:13.579483Z 2021-12-18T23:32:32.543204Z 2021-12-18T23:32:21.337114Z 2021-12-18T23:31:56.358940Z 2021-12-18T23:32:30.070620Z 2021-12-18T23:32:17.350337Z 2021-12-18T23:31:56.809456Z 2021-12-18T23:32:23.196692Z 2021-12-18T23:32:00.526503Z 2021-12-18T23:32:14.977530Z 2021-12-18T23:32:05.528648Z 2021-12-18T23:32:11.145766Z 2021-12-18T23:31:53.785716Z 2021-12-18T23:32:04.991224Z 2021-12-18T23:31:58.489508Z 2021-12-18T23:32:12.569258Z 2021-12-18T23:31:52.561692Z 2021-12-18T23:32:16.226249Z 2021-12-18T23:31:42.922270Z 2021-12-18T23:32:07.593261Z 2021-12-18T23:31:45.348638Z 2021-12-18T23:32:11.740211Z 2021-12-18T23:31:33.700990Z 2021-12-18T23:32:04.609239Z 2021-12-18T23:31:35.685956Z 2021-12-18T23:31:43.552110Z 2021-12-18T23:31:56.070254Z 2021-12-18T23:31:36.586548Z 2021-12-18T23:31:52.457705Z 2021-12-18T23:31:43.864938Z 2021-12-18T23:31:43.245253Z 2021-12-18T23:31:39.891234Z 2021-12-18T23:31:33.682529Z 2021-12-18T23:31:25.454325Z 2021-12-18T23:31:11.254206Z 2021-12-18T23:31:26.752250Z 2021-12-18T23:31:07.844124Z 2021-12-18T23:31:08.174807Z 2021-12-18T23:31:01.361583Z 2021-12-18T23:31:01.908192Z 2021-12-18T23:31:00.799068Z 2021-12-18T23:31:03.672457Z 2021-12-18T23:31:01.815313Z 2021-12-18T23:30:54.067258Z 2021-12-18T23:30:54.113235Z 2021-12-18T23:30:53.477162Z 2021-12-18T23:30:55.846308Z 2021-12-18T23:30:47.685204Z 2021-12-18T23:30:50.034275Z 2021-12-18T23:37:37.423741Z 2021-12-18T23:37:43.598535Z 2021-12-18T23:37:38.825288Z 2021-12-18T23:37:40.405348Z 2021-12-18T23:37:38.579383Z 2021-12-18T23:37:40.307141Z 2021-12-18T23:37:37.470787Z 2021-12-18T23:37:47.442337Z 2021-12-18T23:37:37.772228Z 2021-12-18T23:37:43.226145Z 2021-12-18T23:37:36.142737Z 2021-12-18T23:37:44.884656Z 2021-12-18T23:37:32.858211Z 2021-12-18T23:37:41.306347Z 2021-12-18T23:37:37.046246Z 2021-12-18T23:37:45.376423Z 2021-12-18T23:37:29.561191Z 2021-12-18T23:37:39.324505Z 2021-12-18T23:37:28.591259Z 2021-12-18T23:37:40.698444Z 2021-12-18T23:37:35.226238Z 2021-12-18T23:37:42.246945Z 2021-12-18T23:37:35.377832Z 2021-12-18T23:37:43.106554Z 2021-12-18T23:37:31.895258Z 2021-12-18T23:37:37.669627Z 2021-12-18T23:37:37.039197Z 2021-12-18T23:37:45.053605Z 2021-12-18T23:37:25.706223Z 2021-12-18T23:37:40.142602Z 2021-12-18T23:37:32.290732Z 2021-12-18T23:37:36.364070Z 2021-12-18T23:37:28.912673Z 2021-12-18T23:37:37.457954Z 2021-12-18T23:37:32.290732Z 2021-12-18T23:37:36.510562Z 2021-12-18T23:37:32.872306Z 2021-12-18T23:37:35.778332Z 2021-12-18T23:37:18.931903Z 2021-12-18T23:37:29.589534Z 2021-12-18T23:37:25.899464Z 2021-12-18T23:37:31.489124Z 2021-12-18T23:37:30.047935Z 2021-12-18T23:37:35.660739Z 2021-12-18T23:37:27.959037Z 2021-12-18T23:37:32.922479Z 2021-12-18T23:37:13.344994Z 2021-12-18T23:37:33.313542Z 2021-12-18T23:37:25.085299Z 2021-12-18T23:37:30.818770Z 2021-12-18T23:37:18.368204Z 2021-12-18T23:37:33.037625Z 2021-12-18T23:37:23.499261Z 2021-12-18T23:37:33.476745Z 2021-12-18T23:37:13.607191Z 2021-12-18T23:37:20.991377Z 2021-12-18T23:37:28.278566Z 2021-12-18T23:37:26.553638Z 2021-12-18T23:37:02.227426Z 2021-12-18T23:37:25.826895Z 2021-12-18T23:37:13.997763Z 2021-12-18T23:37:18.138688Z 2021-12-18T23:37:12.231433Z 2021-12-18T23:37:12.609133Z 2021-12-18T23:37:19.122506Z 2021-12-18T23:37:23.223523Z 2021-12-18T23:37:00.466291Z 2021-12-18T23:37:15.153784Z 2021-12-18T23:37:08.939503Z 2021-12-18T23:37:11.769591Z 2021-12-18T23:37:01.322228Z 2021-12-18T23:37:11.812040Z 2021-12-18T23:36:58.697186Z 2021-12-18T23:37:11.900538Z 2021-12-18T23:36:56.392232Z 2021-12-18T23:37:07.795171Z 2021-12-18T23:36:27.380575Z 2021-12-18T23:36:22.242198Z 2021-12-18T23:36:19.734649Z 2021-12-18T23:36:17.906293Z 2021-12-18T23:36:22.570570Z 2021-12-18T23:36:13.440335Z 2021-12-18T23:36:29.086629Z 2021-12-18T23:36:14.233768Z 2021-12-18T23:36:09.796222Z 2021-12-18T23:36:18.465349Z 2021-12-18T23:36:14.953822Z 2021-12-18T23:36:12.460330Z 2021-12-18T23:36:12.280209Z 2021-12-18T23:36:08.701224Z 2021-12-18T23:36:06.922515Z 2021-12-18T23:36:05.194173Z 2021-12-18T23:36:02.532547Z 2021-12-18T23:36:06.399838Z 2021-12-18T23:36:00.481587Z 2021-12-18T23:36:03.147306Z 2021-12-18T23:35:48.224699Z 2021-12-18T23:35:50.499448Z 2021-12-18T23:35:47.416226Z 2021-12-18T23:35:47.824741Z 2021-12-18T23:42:45.369084Z 2021-12-18T23:42:43.158391Z 2021-12-18T23:42:41.879608Z 2021-12-18T23:42:48.736851Z 2021-12-18T23:42:38.677416Z 2021-12-18T23:42:39.796992Z 2021-12-18T23:42:37.606791Z 2021-12-18T23:42:47.173638Z 2021-12-18T23:42:41.485490Z 2021-12-18T23:42:48.327675Z 2021-12-18T23:42:41.776375Z 2021-12-18T23:42:47.620744Z 2021-12-18T23:42:42.695710Z 2021-12-18T23:42:40.232622Z 2021-12-18T23:42:47.092709Z 2021-12-18T23:42:36.492544Z 2021-12-18T23:42:43.034214Z 2021-12-18T23:42:44.307066Z 2021-12-18T23:42:39.967983Z 2021-12-18T23:42:44.910584Z 2021-12-18T23:42:41.941599Z 2021-12-18T23:42:42.541110Z 2021-12-18T23:42:37.673103Z 2021-12-18T23:42:45.574217Z 2021-12-18T23:42:38.222545Z 2021-12-18T23:42:44.376328Z 2021-12-18T23:42:37.426369Z 2021-12-18T23:42:43.735510Z 2021-12-18T23:42:40.237663Z 2021-12-18T23:42:45.398253Z 2021-12-18T23:42:38.431112Z 2021-12-18T23:42:35.343664Z 2021-12-18T23:42:32.663184Z 2021-12-18T23:42:44.256500Z 2021-12-18T23:42:41.278570Z 2021-12-18T23:42:39.067191Z 2021-12-18T23:42:33.980566Z 2021-12-18T23:42:38.155790Z 2021-12-18T23:42:31.351280Z 2021-12-18T23:42:40.028245Z 2021-12-18T23:42:32.841512Z 2021-12-18T23:42:35.731121Z 2021-12-18T23:42:33.936172Z 2021-12-18T23:42:33.253321Z 2021-12-18T23:42:34.344337Z 2021-12-18T23:42:39.171346Z 2021-12-18T23:42:28.475707Z 2021-12-18T23:42:19.283204Z 2021-12-18T23:42:17.431913Z 2021-12-18T23:42:27.090334Z 2021-12-18T23:42:26.328623Z 2021-12-18T23:42:31.826595Z 2021-12-18T23:42:26.376450Z 2021-12-18T23:42:27.376891Z 2021-12-18T23:42:21.910448Z 2021-12-18T23:42:25.485354Z 2021-12-18T23:42:25.613408Z 2021-12-18T23:42:29.852744Z 2021-12-18T23:42:21.790432Z 2021-12-18T23:42:32.891232Z 2021-12-18T23:42:14.923891Z 2021-12-18T23:42:20.448266Z 2021-12-18T23:42:10.457836Z 2021-12-18T23:42:26.883235Z 2021-12-18T23:42:20.625118Z 2021-12-18T23:42:27.404542Z 2021-12-18T23:42:16.315969Z 2021-12-18T23:42:22.755146Z 2021-12-18T23:42:16.433191Z 2021-12-18T23:42:15.722277Z 2021-12-18T23:42:16.576666Z 2021-12-18T23:42:21.928763Z 2021-12-18T23:42:03.073513Z 2021-12-18T23:42:17.161351Z 2021-12-18T23:41:46.460600Z 2021-12-18T23:42:10.032214Z 2021-12-18T23:41:57.375348Z 2021-12-18T23:41:48.200010Z 2021-12-18T23:41:40.709580Z 2021-12-18T23:41:40.069452Z 2021-12-18T23:41:26.201410Z 2021-12-18T23:41:44.763072Z 2021-12-18T23:41:36.476804Z 2021-12-18T23:41:21.296916Z 2021-12-18T23:41:49.845939Z 2021-12-18T23:41:35.017375Z 2021-12-18T23:41:21.784439Z 2021-12-18T23:41:18.834367Z 2021-12-18T23:41:20.832598Z 2021-12-18T23:41:25.370235Z 2021-12-18T23:41:11.880495Z 2021-12-18T23:41:21.016187Z 2021-12-18T23:41:06.653149Z 2021-12-18T23:41:21.850570Z 2021-12-18T23:41:05.858595Z 2021-12-18T23:41:04.121795Z 2021-12-18T23:40:51.997593Z 2021-12-18T23:40:52.899593Z 2021-12-18T23:40:48.404760Z 2021-12-18T23:40:45.805474Z 2021-12-18T23:47:48.564055Z 2021-12-18T23:47:45.549959Z 2021-12-18T23:47:46.309857Z 2021-12-18T23:47:48.253034Z 2021-12-18T23:47:45.089672Z 2021-12-18T23:47:46.309001Z 2021-12-18T23:47:48.716765Z 2021-12-18T23:47:48.740535Z 2021-12-18T23:47:48.553244Z 2021-12-18T23:47:47.572350Z 2021-12-18T23:47:45.714488Z 2021-12-18T23:47:48.469995Z 2021-12-18T23:47:48.410458Z 2021-12-18T23:47:46.073125Z 2021-12-18T23:47:43.078221Z 2021-12-18T23:47:46.182548Z 2021-12-18T23:47:45.423004Z 2021-12-18T23:47:51.193205Z 2021-12-18T23:47:41.672259Z 2021-12-18T23:47:45.660601Z 2021-12-18T23:47:40.018809Z 2021-12-18T23:47:43.672886Z 2021-12-18T23:47:44.398649Z 2021-12-18T23:47:39.418374Z 2021-12-18T23:47:41.264262Z 2021-12-18T23:47:40.912544Z 2021-12-18T23:47:38.138657Z 2021-12-18T23:47:40.787146Z 2021-12-18T23:47:39.955194Z 2021-12-18T23:47:35.008507Z 2021-12-18T23:47:30.396198Z 2021-12-18T23:47:40.793570Z 2021-12-18T23:47:42.011600Z 2021-12-18T23:47:39.041174Z 2021-12-18T23:47:37.661979Z 2021-12-18T23:47:42.635569Z 2021-12-18T23:47:28.238163Z 2021-12-18T23:47:39.323484Z 2021-12-18T23:47:33.960089Z 2021-12-18T23:47:35.032580Z 2021-12-18T23:47:36.296023Z 2021-12-18T23:47:29.892118Z 2021-12-18T23:47:27.700284Z 2021-12-18T23:47:33.527405Z 2021-12-18T23:47:26.546243Z 2021-12-18T23:47:22.052785Z 2021-12-18T23:47:34.178505Z 2021-12-18T23:47:30.174526Z 2021-12-18T23:47:27.420276Z 2021-12-18T23:47:26.168613Z 2021-12-18T23:47:24.028155Z 2021-12-18T23:47:25.514600Z 2021-12-18T23:47:28.339576Z 2021-12-18T23:46:54.691284Z 2021-12-18T23:46:53.349078Z 2021-12-18T23:46:56.045462Z 2021-12-18T23:46:44.923585Z 2021-12-18T23:46:49.868636Z 2021-12-18T23:46:52.722212Z 2021-12-18T23:46:48.348697Z 2021-12-18T23:46:51.753774Z 2021-12-18T23:46:53.829198Z 2021-12-18T23:46:48.793657Z 2021-12-18T23:46:47.602269Z 2021-12-18T23:46:45.498541Z 2021-12-18T23:46:52.715304Z 2021-12-18T23:46:45.544428Z 2021-12-18T23:46:50.162936Z 2021-12-18T23:46:48.768660Z 2021-12-18T23:46:44.837386Z 2021-12-18T23:46:42.338985Z 2021-12-18T23:46:42.478831Z 2021-12-18T23:46:42.133135Z 2021-12-18T23:46:47.099174Z 2021-12-18T23:46:38.697413Z 2021-12-18T23:46:36.247198Z 2021-12-18T23:46:42.099580Z 2021-12-18T23:46:39.765222Z 2021-12-18T23:46:40.662401Z 2021-12-18T23:46:39.042408Z 2021-12-18T23:46:36.605585Z 2021-12-18T23:46:41.344229Z 2021-12-18T23:46:29.616638Z 2021-12-18T23:46:29.960025Z 2021-12-18T23:46:27.328273Z 2021-12-18T23:46:35.860686Z 2021-12-18T23:46:34.094369Z 2021-12-18T23:46:37.409200Z 2021-12-18T23:46:27.144833Z 2021-12-18T23:46:33.808846Z 2021-12-18T23:46:29.137125Z 2021-12-18T23:46:35.220991Z 2021-12-18T23:46:27.146671Z 2021-12-18T23:46:28.793014Z 2021-12-18T23:46:24.145758Z 2021-12-18T23:46:00.809653Z 2021-12-18T23:46:25.177206Z 2021-12-18T23:46:02.012180Z 2021-12-18T23:45:57.316388Z 2021-12-18T23:45:55.650084Z", + "failed_job_ids": " [196]", + "finished": "2021-12-18T23:27:01.064779Z 2021-12-18T23:26:26.938024Z 2021-12-18T23:26:59.618253Z 2021-12-18T23:26:19.461271Z 2021-12-18T23:26:58.828040Z 2021-12-18T23:26:05.474775Z 2021-12-18T23:26:40.699525Z 2021-12-18T23:26:03.910418Z 2021-12-18T23:26:47.289220Z 2021-12-18T23:25:57.309939Z 2021-12-18T23:26:36.513113Z 2021-12-18T23:26:43.668215Z 2021-12-18T23:28:14.995821Z 2021-12-18T23:25:58.294554Z 2021-12-18T23:26:30.820268Z 2021-12-18T23:26:02.442071Z 2021-12-18T23:27:38.638814Z 2021-12-18T23:25:59.683109Z 2021-12-18T23:27:19.606871Z 2021-12-18T23:26:01.006115Z 2021-12-18T23:27:55.911755Z 2021-12-18T23:25:53.189747Z 2021-12-18T23:27:47.584641Z 2021-12-18T23:25:52.597232Z 2021-12-18T23:27:11.064513Z 2021-12-18T23:25:48.610535Z 2021-12-18T23:28:05.676512Z 2021-12-18T23:25:45.718167Z 2021-12-18T23:25:47.287768Z 2021-12-18T23:25:38.530224Z 2021-12-18T23:27:02.023671Z 2021-12-18T23:25:38.582710Z 2021-12-18T23:27:29.981581Z 2021-12-18T23:25:34.516111Z 2021-12-18T23:25:37.593345Z 2021-12-18T23:25:26.432575Z 2021-12-18T23:27:15.863314Z 2021-12-18T23:26:58.260957Z 2021-12-18T23:27:16.429591Z 2021-12-18T23:26:57.926334Z 2021-12-18T23:27:17.224517Z 2021-12-18T23:26:59.434892Z 2021-12-18T23:27:16.089293Z 2021-12-18T23:26:59.131379Z 2021-12-18T23:27:14.500497Z 2021-12-18T23:26:58.964457Z 2021-12-18T23:27:15.632686Z 2021-12-18T23:26:54.992090Z 2021-12-18T23:27:16.713429Z 2021-12-18T23:26:58.755442Z 2021-12-18T23:27:13.221556Z 2021-12-18T23:26:55.904712Z 2021-12-18T23:27:15.092446Z 2021-12-18T23:26:53.702359Z 2021-12-18T23:27:09.821205Z 2021-12-18T23:26:57.889945Z 2021-12-18T23:27:10.954725Z 2021-12-18T23:26:57.133038Z 2021-12-18T23:27:17.513073Z 2021-12-18T23:27:16.679372Z 2021-12-18T23:27:13.357590Z 2021-12-18T23:27:17.267767Z 2021-12-18T23:27:17.277298Z 2021-12-18T23:27:02.909636Z 2021-12-18T23:26:52.346926Z 2021-12-18T23:27:12.118066Z 2021-12-18T23:26:53.937810Z 2021-12-18T23:27:07.431024Z 2021-12-18T23:26:50.882134Z 2021-12-18T23:27:09.301110Z 2021-12-18T23:26:51.682370Z 2021-12-18T23:27:11.073149Z 2021-12-18T23:26:45.689189Z 2021-12-18T23:27:06.957052Z 2021-12-18T23:26:50.622202Z 2021-12-18T23:27:09.466528Z 2021-12-18T23:27:01.405794Z 2021-12-18T23:26:42.825054Z 2021-12-18T23:27:01.217261Z 2021-12-18T23:26:46.798834Z 2021-12-18T23:27:09.420366Z 2021-12-18T23:26:49.847287Z 2021-12-18T23:27:02.295507Z 2021-12-18T23:26:46.021360Z 2021-12-18T23:27:03.288198Z 2021-12-18T23:26:44.529265Z 2021-12-18T23:27:02.226230Z 2021-12-18T23:26:46.618215Z 2021-12-18T23:26:58.628397Z 2021-12-18T23:26:34.729633Z 2021-12-18T23:26:57.399871Z 2021-12-18T23:26:33.462320Z 2021-12-18T23:26:47.895455Z 2021-12-18T23:26:37.588290Z 2021-12-18T23:26:56.316300Z 2021-12-18T23:26:37.419881Z 2021-12-18T23:26:40.778062Z 2021-12-18T23:26:36.435902Z 2021-12-18T23:27:08.759127Z 2021-12-18T23:26:35.106271Z 2021-12-18T23:32:32.494733Z 2021-12-18T23:32:48.098999Z 2021-12-18T23:32:31.630658Z 2021-12-18T23:32:49.088179Z 2021-12-18T23:32:28.610160Z 2021-12-18T23:32:46.330312Z 2021-12-18T23:32:32.038081Z 2021-12-18T23:32:50.482291Z 2021-12-18T23:32:29.132711Z 2021-12-18T23:32:49.725218Z 2021-12-18T23:32:32.907274Z 2021-12-18T23:32:49.685838Z 2021-12-18T23:32:22.047128Z 2021-12-18T23:32:51.250660Z 2021-12-18T23:32:31.936637Z 2021-12-18T23:32:40.669562Z 2021-12-18T23:32:31.681623Z 2021-12-18T23:32:51.287022Z 2021-12-18T23:32:46.773506Z 2021-12-18T23:32:31.675835Z 2021-12-18T23:32:26.718802Z 2021-12-18T23:32:50.641546Z 2021-12-18T23:32:29.992116Z 2021-12-18T23:32:44.037631Z 2021-12-18T23:32:26.982478Z 2021-12-18T23:32:49.472901Z 2021-12-18T23:32:24.132794Z 2021-12-18T23:32:39.317574Z 2021-12-18T23:32:29.413769Z 2021-12-18T23:32:51.345775Z 2021-12-18T23:32:28.804563Z 2021-12-18T23:32:47.333076Z 2021-12-18T23:32:27.017140Z 2021-12-18T23:32:45.770877Z 2021-12-18T23:32:24.260221Z 2021-12-18T23:32:41.439014Z 2021-12-18T23:32:20.725866Z 2021-12-18T23:32:35.366229Z 2021-12-18T23:32:25.750721Z 2021-12-18T23:32:45.168065Z 2021-12-18T23:32:25.491146Z 2021-12-18T23:32:49.188400Z 2021-12-18T23:32:27.903469Z 2021-12-18T23:32:40.442649Z 2021-12-18T23:32:22.126774Z 2021-12-18T23:32:38.374311Z 2021-12-18T23:32:18.266217Z 2021-12-18T23:32:40.245229Z 2021-12-18T23:32:20.967923Z 2021-12-18T23:32:33.159423Z 2021-12-18T23:32:18.024577Z 2021-12-18T23:32:37.935844Z 2021-12-18T23:32:27.481219Z 2021-12-18T23:32:00.771316Z 2021-12-18T23:32:35.320080Z 2021-12-18T23:32:24.082262Z 2021-12-18T23:32:02.893395Z 2021-12-18T23:32:27.784060Z 2021-12-18T23:32:05.732986Z 2021-12-18T23:32:20.610044Z 2021-12-18T23:32:10.639343Z 2021-12-18T23:32:17.001182Z 2021-12-18T23:31:59.149263Z 2021-12-18T23:32:10.995558Z 2021-12-18T23:32:04.713321Z 2021-12-18T23:32:17.914997Z 2021-12-18T23:32:00.107515Z 2021-12-18T23:32:22.270227Z 2021-12-18T23:31:48.607298Z 2021-12-18T23:32:13.080490Z 2021-12-18T23:31:52.319529Z 2021-12-18T23:32:17.698060Z 2021-12-18T23:31:41.315851Z 2021-12-18T23:32:11.217277Z 2021-12-18T23:31:41.404504Z 2021-12-18T23:31:04.453840Z 2021-12-18T23:31:48.473322Z 2021-12-18T23:32:02.331285Z 2021-12-18T23:31:40.764382Z 2021-12-18T23:32:01.103647Z 2021-12-18T23:31:49.142398Z 2021-12-18T23:31:49.409895Z 2021-12-18T23:31:46.886936Z 2021-12-18T23:31:38.752402Z 2021-12-18T23:31:30.721173Z 2021-12-18T23:31:16.084672Z 2021-12-18T23:31:34.400703Z 2021-12-18T23:31:12.447821Z 2021-12-18T23:31:13.459146Z 2021-12-18T23:31:05.745191Z 2021-12-18T23:31:07.296529Z 2021-12-18T23:31:05.340937Z 2021-12-18T23:31:07.813685Z 2021-12-18T23:31:06.077803Z 2021-12-18T23:30:58.100872Z 2021-12-18T23:30:58.236841Z 2021-12-18T23:30:58.074608Z 2021-12-18T23:31:00.008994Z 2021-12-18T23:30:52.176884Z 2021-12-18T23:30:53.017251Z 2021-12-18T23:37:42.617244Z 2021-12-18T23:37:48.009229Z 2021-12-18T23:37:42.413887Z 2021-12-18T23:37:44.768788Z 2021-12-18T23:37:42.285613Z 2021-12-18T23:37:45.773198Z 2021-12-18T23:37:42.004043Z 2021-12-18T23:37:49.566266Z 2021-12-18T23:37:41.516752Z 2021-12-18T23:37:47.776400Z 2021-12-18T23:37:40.674617Z 2021-12-18T23:37:48.665588Z 2021-12-18T23:37:39.198780Z 2021-12-18T23:37:46.962360Z 2021-12-18T23:37:41.518631Z 2021-12-18T23:37:49.650153Z 2021-12-18T23:37:34.978647Z 2021-12-18T23:37:44.038486Z 2021-12-18T23:37:34.155901Z 2021-12-18T23:37:45.530155Z 2021-12-18T23:37:40.023791Z 2021-12-18T23:37:47.738175Z 2021-12-18T23:37:39.703744Z 2021-12-18T23:37:48.170304Z 2021-12-18T23:37:37.918391Z 2021-12-18T23:37:43.335301Z 2021-12-18T23:37:41.520996Z 2021-12-18T23:37:48.533069Z 2021-12-18T23:37:33.096243Z 2021-12-18T23:37:45.952979Z 2021-12-18T23:37:37.445505Z 2021-12-18T23:37:42.307681Z 2021-12-18T23:37:33.287988Z 2021-12-18T23:37:41.961588Z 2021-12-18T23:37:38.023451Z 2021-12-18T23:37:43.569885Z 2021-12-18T23:37:37.553190Z 2021-12-18T23:37:41.515706Z 2021-12-18T23:37:25.181848Z 2021-12-18T23:37:34.677565Z 2021-12-18T23:37:31.035213Z 2021-12-18T23:37:38.474612Z 2021-12-18T23:37:35.087530Z 2021-12-18T23:37:41.918115Z 2021-12-18T23:37:34.497630Z 2021-12-18T23:37:40.101228Z 2021-12-18T23:37:19.221336Z 2021-12-18T23:37:42.344527Z 2021-12-18T23:37:30.153776Z 2021-12-18T23:37:36.674750Z 2021-12-18T23:37:24.903246Z 2021-12-18T23:37:41.505439Z 2021-12-18T23:37:29.090300Z 2021-12-18T23:37:37.283636Z 2021-12-18T23:37:19.528332Z 2021-12-18T23:37:26.681341Z 2021-12-18T23:37:34.492148Z 2021-12-18T23:37:32.224674Z 2021-12-18T23:37:08.254329Z 2021-12-18T23:37:31.933572Z 2021-12-18T23:37:19.370519Z 2021-12-18T23:37:24.256300Z 2021-12-18T23:37:19.048440Z 2021-12-18T23:37:19.067089Z 2021-12-18T23:37:24.683465Z 2021-12-18T23:37:28.527611Z 2021-12-18T23:37:05.022371Z 2021-12-18T23:37:21.674944Z 2021-12-18T23:37:16.451556Z 2021-12-18T23:37:19.288663Z 2021-12-18T23:37:05.349857Z 2021-12-18T23:37:18.241635Z 2021-12-18T23:37:04.716363Z 2021-12-18T23:37:18.733463Z 2021-12-18T23:37:02.652083Z 2021-12-18T23:37:13.899390Z 2021-12-18T23:36:33.068745Z 2021-12-18T23:36:27.302128Z 2021-12-18T23:36:26.844576Z 2021-12-18T23:36:23.749032Z 2021-12-18T23:36:29.166353Z 2021-12-18T23:36:19.190306Z 2021-12-18T23:36:35.515710Z 2021-12-18T23:36:19.370420Z 2021-12-18T23:36:14.382730Z 2021-12-18T23:36:22.940261Z 2021-12-18T23:36:21.086737Z 2021-12-18T23:36:17.378541Z 2021-12-18T23:36:17.144508Z 2021-12-18T23:36:13.426558Z 2021-12-18T23:36:09.170623Z 2021-12-18T23:36:10.219506Z 2021-12-18T23:36:07.289966Z 2021-12-18T23:36:10.569285Z 2021-12-18T23:36:05.215055Z 2021-12-18T23:36:07.618226Z 2021-12-18T23:35:51.657166Z 2021-12-18T23:35:54.895497Z 2021-12-18T23:35:51.483555Z 2021-12-18T23:35:52.086412Z 2021-12-18T23:42:47.788531Z 2021-12-18T23:42:47.922316Z 2021-12-18T23:42:46.689434Z 2021-12-18T23:42:52.903270Z 2021-12-18T23:42:44.069885Z 2021-12-18T23:42:45.123633Z 2021-12-18T23:42:42.520528Z 2021-12-18T23:42:51.720560Z 2021-12-18T23:42:47.199645Z 2021-12-18T23:42:53.185568Z 2021-12-18T23:42:47.211307Z 2021-12-18T23:42:52.618046Z 2021-12-18T23:42:47.002536Z 2021-12-18T23:42:47.626164Z 2021-12-18T23:42:52.162401Z 2021-12-18T23:42:41.040192Z 2021-12-18T23:42:47.042795Z 2021-12-18T23:42:49.841064Z 2021-12-18T23:42:44.951211Z 2021-12-18T23:42:49.647474Z 2021-12-18T23:42:47.335225Z 2021-12-18T23:42:47.509033Z 2021-12-18T23:42:43.097037Z 2021-12-18T23:42:50.334499Z 2021-12-18T23:42:45.193268Z 2021-12-18T23:42:48.920938Z 2021-12-18T23:42:42.582626Z 2021-12-18T23:42:49.607350Z 2021-12-18T23:42:44.721959Z 2021-12-18T23:42:50.484209Z 2021-12-18T23:42:44.211981Z 2021-12-18T23:42:41.701061Z 2021-12-18T23:42:38.200697Z 2021-12-18T23:42:48.748304Z 2021-12-18T23:42:46.251620Z 2021-12-18T23:42:46.062071Z 2021-12-18T23:42:39.449393Z 2021-12-18T23:42:44.161830Z 2021-12-18T23:42:35.689439Z 2021-12-18T23:42:44.724519Z 2021-12-18T23:42:36.971797Z 2021-12-18T23:42:41.318556Z 2021-12-18T23:42:40.909305Z 2021-12-18T23:42:38.313428Z 2021-12-18T23:42:39.252991Z 2021-12-18T23:42:45.604573Z 2021-12-18T23:42:34.095666Z 2021-12-18T23:42:25.841253Z 2021-12-18T23:42:22.766829Z 2021-12-18T23:42:35.089851Z 2021-12-18T23:42:32.869757Z 2021-12-18T23:42:36.602385Z 2021-12-18T23:42:31.932502Z 2021-12-18T23:42:33.818189Z 2021-12-18T23:42:28.619686Z 2021-12-18T23:42:31.423264Z 2021-12-18T23:42:31.084665Z 2021-12-18T23:42:35.346384Z 2021-12-18T23:42:26.717320Z 2021-12-18T23:42:39.142127Z 2021-12-18T23:42:19.375040Z 2021-12-18T23:42:25.673797Z 2021-12-18T23:42:16.248032Z 2021-12-18T23:42:33.393124Z 2021-12-18T23:42:26.949444Z 2021-12-18T23:42:33.910194Z 2021-12-18T23:42:21.025209Z 2021-12-18T23:42:30.296348Z 2021-12-18T23:42:22.512563Z 2021-12-18T23:42:23.388015Z 2021-12-18T23:42:23.262650Z 2021-12-18T23:42:27.995404Z 2021-12-18T23:42:11.279903Z 2021-12-18T23:42:24.422109Z 2021-12-18T23:41:54.237160Z 2021-12-18T23:42:15.411838Z 2021-12-18T23:42:02.683171Z 2021-12-18T23:41:54.797352Z 2021-12-18T23:41:44.751215Z 2021-12-18T23:41:45.187621Z 2021-12-18T23:41:31.307227Z 2021-12-18T23:41:52.708266Z 2021-12-18T23:41:44.682838Z 2021-12-18T23:41:28.698656Z 2021-12-18T23:41:57.944458Z 2021-12-18T23:41:43.499996Z 2021-12-18T23:41:27.866602Z 2021-12-18T23:41:25.284873Z 2021-12-18T23:41:27.332208Z 2021-12-18T23:41:31.886315Z 2021-12-18T23:41:17.248515Z 2021-12-18T23:41:28.260814Z 2021-12-18T23:41:10.130044Z 2021-12-18T23:41:29.584668Z 2021-12-18T23:41:11.210461Z 2021-12-18T23:41:08.464312Z 2021-12-18T23:40:56.237100Z 2021-12-18T23:40:56.985108Z 2021-12-18T23:40:52.729013Z 2021-12-18T23:40:49.236605Z 2021-12-18T23:47:52.816343Z 2021-12-18T23:47:50.104577Z 2021-12-18T23:47:50.768629Z 2021-12-18T23:47:52.404794Z 2021-12-18T23:47:50.588609Z 2021-12-18T23:47:51.757025Z 2021-12-18T23:47:53.364810Z 2021-12-18T23:47:52.682971Z 2021-12-18T23:47:52.428950Z 2021-12-18T23:47:51.861641Z 2021-12-18T23:47:50.861397Z 2021-12-18T23:47:52.165120Z 2021-12-18T23:47:52.168132Z 2021-12-18T23:47:50.006590Z 2021-12-18T23:47:47.916781Z 2021-12-18T23:47:50.749407Z 2021-12-18T23:47:50.327823Z 2021-12-18T23:47:52.547094Z 2021-12-18T23:47:46.230978Z 2021-12-18T23:47:49.823090Z 2021-12-18T23:47:44.898387Z 2021-12-18T23:47:48.426330Z 2021-12-18T23:47:49.124488Z 2021-12-18T23:47:46.036579Z 2021-12-18T23:47:45.394700Z 2021-12-18T23:47:45.381651Z 2021-12-18T23:47:44.428833Z 2021-12-18T23:47:45.954678Z 2021-12-18T23:47:46.173900Z 2021-12-18T23:47:40.381279Z 2021-12-18T23:47:35.914337Z 2021-12-18T23:47:45.747365Z 2021-12-18T23:47:47.255318Z 2021-12-18T23:47:42.924266Z 2021-12-18T23:47:42.521979Z 2021-12-18T23:47:47.720614Z 2021-12-18T23:47:34.668789Z 2021-12-18T23:47:44.460533Z 2021-12-18T23:47:40.344731Z 2021-12-18T23:47:39.068280Z 2021-12-18T23:47:41.222321Z 2021-12-18T23:47:35.596284Z 2021-12-18T23:47:33.333096Z 2021-12-18T23:47:37.800029Z 2021-12-18T23:47:31.746900Z 2021-12-18T23:47:27.433612Z 2021-12-18T23:47:39.894147Z 2021-12-18T23:47:35.593004Z 2021-12-18T23:47:32.898953Z 2021-12-18T23:47:30.419404Z 2021-12-18T23:47:29.798244Z 2021-12-18T23:47:31.155825Z 2021-12-18T23:47:33.717374Z 2021-12-18T23:47:00.009358Z 2021-12-18T23:46:59.004327Z 2021-12-18T23:47:01.765196Z 2021-12-18T23:46:50.612271Z 2021-12-18T23:46:55.190762Z 2021-12-18T23:46:59.314683Z 2021-12-18T23:46:54.732469Z 2021-12-18T23:46:57.362510Z 2021-12-18T23:46:59.481752Z 2021-12-18T23:46:54.232700Z 2021-12-18T23:46:53.397201Z 2021-12-18T23:46:51.190341Z 2021-12-18T23:46:57.724211Z 2021-12-18T23:46:50.098518Z 2021-12-18T23:46:56.955585Z 2021-12-18T23:46:53.926617Z 2021-12-18T23:46:50.028244Z 2021-12-18T23:46:46.882936Z 2021-12-18T23:46:46.896319Z 2021-12-18T23:46:47.165967Z 2021-12-18T23:46:52.767187Z 2021-12-18T23:46:43.653817Z 2021-12-18T23:46:41.156803Z 2021-12-18T23:46:47.703832Z 2021-12-18T23:46:44.518687Z 2021-12-18T23:46:46.817307Z 2021-12-18T23:46:44.813150Z 2021-12-18T23:46:42.058618Z 2021-12-18T23:46:46.533521Z 2021-12-18T23:46:35.069377Z 2021-12-18T23:46:34.820718Z 2021-12-18T23:46:32.758313Z 2021-12-18T23:46:40.177502Z 2021-12-18T23:46:39.126773Z 2021-12-18T23:46:41.888829Z 2021-12-18T23:46:31.579855Z 2021-12-18T23:46:38.923778Z 2021-12-18T23:46:33.551737Z 2021-12-18T23:46:39.548345Z 2021-12-18T23:46:32.703754Z 2021-12-18T23:46:34.403690Z 2021-12-18T23:46:29.427172Z 2021-12-18T23:46:04.790493Z 2021-12-18T23:46:30.986561Z 2021-12-18T23:46:06.499689Z 2021-12-18T23:46:01.408060Z 2021-12-18T23:45:59.453476Z", + "finished_diff": 128.75180640000002, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 167.59042, + "mean": 60.41240077555112, + "min": 5.49353 + }, + "jobs_events_duration": { + "max": 10.518678, + "mean": 3.856368406813627, + "min": 0.188147 + }, + "jobs_events_lag": { + "max": 419.32745, + "mean": -3.41609546492986, + "min": -46.518815 + }, + "jobs_waiting": { + "max": 69.093097, + "mean": 31.020942006012046, + "min": 0.598929 + }, + "modified": "2021-12-18T23:25:36.352840Z 2021-12-18T23:25:36.179293Z 2021-12-18T23:25:36.124175Z 2021-12-18T23:25:36.041043Z 2021-12-18T23:25:35.935814Z 2021-12-18T23:25:35.821054Z 2021-12-18T23:25:35.533618Z 2021-12-18T23:25:35.218674Z 2021-12-18T23:25:35.077455Z 2021-12-18T23:25:34.899345Z 2021-12-18T23:25:34.764427Z 2021-12-18T23:25:34.629827Z 2021-12-18T23:25:21.862984Z 2021-12-18T23:25:21.620258Z 2021-12-18T23:25:21.519202Z 2021-12-18T23:25:21.346896Z 2021-12-18T23:25:21.185632Z 2021-12-18T23:25:21.097477Z 2021-12-18T23:25:20.970657Z 2021-12-18T23:25:20.897325Z 2021-12-18T23:25:20.826717Z 2021-12-18T23:25:20.758622Z 2021-12-18T23:25:20.699717Z 2021-12-18T23:25:20.616312Z 2021-12-18T23:25:20.578378Z 2021-12-18T23:25:20.481030Z 2021-12-18T23:25:20.363442Z 2021-12-18T23:25:17.866331Z 2021-12-18T23:25:17.734240Z 2021-12-18T23:25:17.686424Z 2021-12-18T23:25:17.626828Z 2021-12-18T23:25:16.573517Z 2021-12-18T23:25:16.529797Z 2021-12-18T23:25:15.901240Z 2021-12-18T23:25:15.857866Z 2021-12-18T23:25:14.398797Z 2021-12-18T23:26:04.102144Z 2021-12-18T23:26:04.033749Z 2021-12-18T23:26:03.971512Z 2021-12-18T23:26:03.917338Z 2021-12-18T23:26:03.871302Z 2021-12-18T23:26:03.805369Z 2021-12-18T23:26:03.738048Z 2021-12-18T23:26:03.666870Z 2021-12-18T23:26:03.596621Z 2021-12-18T23:26:03.533445Z 2021-12-18T23:26:03.469586Z 2021-12-18T23:26:03.388433Z 2021-12-18T23:26:03.308393Z 2021-12-18T23:26:03.253280Z 2021-12-18T23:26:03.202495Z 2021-12-18T23:26:03.141605Z 2021-12-18T23:26:03.074207Z 2021-12-18T23:26:02.993592Z 2021-12-18T23:26:02.910445Z 2021-12-18T23:26:02.842649Z 2021-12-18T23:26:02.770473Z 2021-12-18T23:26:02.694587Z 2021-12-18T23:26:02.613539Z 2021-12-18T23:26:02.530610Z 2021-12-18T23:26:02.453929Z 2021-12-18T23:26:02.347745Z 2021-12-18T23:26:02.262637Z 2021-12-18T23:25:42.061136Z 2021-12-18T23:25:42.000213Z 2021-12-18T23:25:41.952743Z 2021-12-18T23:25:41.856094Z 2021-12-18T23:25:41.743605Z 2021-12-18T23:25:41.602942Z 2021-12-18T23:25:41.466228Z 2021-12-18T23:25:41.326204Z 2021-12-18T23:25:41.180362Z 2021-12-18T23:25:41.054236Z 2021-12-18T23:25:40.896236Z 2021-12-18T23:25:40.737663Z 2021-12-18T23:26:02.196552Z 2021-12-18T23:25:40.548203Z 2021-12-18T23:25:40.384255Z 2021-12-18T23:25:40.248967Z 2021-12-18T23:25:39.995036Z 2021-12-18T23:25:39.860979Z 2021-12-18T23:25:39.671401Z 2021-12-18T23:25:39.592226Z 2021-12-18T23:25:39.313042Z 2021-12-18T23:25:39.134030Z 2021-12-18T23:25:38.965223Z 2021-12-18T23:25:38.807218Z 2021-12-18T23:25:38.647765Z 2021-12-18T23:25:38.445703Z 2021-12-18T23:25:38.310016Z 2021-12-18T23:25:38.275403Z 2021-12-18T23:25:38.181427Z 2021-12-18T23:25:38.084261Z 2021-12-18T23:25:38.005237Z 2021-12-18T23:25:37.907316Z 2021-12-18T23:25:37.750469Z 2021-12-18T23:25:37.532520Z 2021-12-18T23:25:37.250865Z 2021-12-18T23:25:37.029766Z 2021-12-18T23:25:36.717396Z 2021-12-18T23:31:15.062265Z 2021-12-18T23:31:14.968514Z 2021-12-18T23:31:14.889043Z 2021-12-18T23:31:14.808441Z 2021-12-18T23:31:14.725107Z 2021-12-18T23:31:14.640584Z 2021-12-18T23:31:14.549503Z 2021-12-18T23:31:14.459291Z 2021-12-18T23:31:14.343782Z 2021-12-18T23:31:14.231240Z 2021-12-18T23:31:14.128288Z 2021-12-18T23:31:14.029866Z 2021-12-18T23:31:13.970459Z 2021-12-18T23:31:13.904241Z 2021-12-18T23:31:13.838279Z 2021-12-18T23:31:13.777690Z 2021-12-18T23:31:13.715834Z 2021-12-18T23:31:13.664164Z 2021-12-18T23:31:13.564273Z 2021-12-18T23:31:13.614087Z 2021-12-18T23:31:13.508326Z 2021-12-18T23:31:13.448246Z 2021-12-18T23:31:13.376074Z 2021-12-18T23:31:13.300299Z 2021-12-18T23:31:13.240115Z 2021-12-18T23:31:13.182383Z 2021-12-18T23:31:13.122468Z 2021-12-18T23:31:13.056804Z 2021-12-18T23:31:13.001234Z 2021-12-18T23:31:12.929692Z 2021-12-18T23:31:12.865252Z 2021-12-18T23:31:12.740243Z 2021-12-18T23:31:12.641383Z 2021-12-18T23:31:12.520564Z 2021-12-18T23:31:12.404685Z 2021-12-18T23:31:12.314349Z 2021-12-18T23:31:12.218533Z 2021-12-18T23:31:12.126259Z 2021-12-18T23:31:12.006564Z 2021-12-18T23:31:11.922896Z 2021-12-18T23:31:11.833689Z 2021-12-18T23:31:11.752035Z 2021-12-18T23:31:11.647683Z 2021-12-18T23:31:11.550533Z 2021-12-18T23:31:11.454577Z 2021-12-18T23:31:11.358471Z 2021-12-18T23:31:11.251292Z 2021-12-18T23:31:11.160256Z 2021-12-18T23:31:11.064341Z 2021-12-18T23:31:10.952398Z 2021-12-18T23:31:10.787991Z 2021-12-18T23:31:10.649573Z 2021-12-18T23:30:55.048208Z 2021-12-18T23:30:54.883754Z 2021-12-18T23:31:10.482247Z 2021-12-18T23:30:54.721884Z 2021-12-18T23:30:54.531787Z 2021-12-18T23:30:54.357089Z 2021-12-18T23:30:54.216865Z 2021-12-18T23:30:53.882119Z 2021-12-18T23:30:53.572507Z 2021-12-18T23:30:53.097153Z 2021-12-18T23:30:52.555399Z 2021-12-18T23:30:52.311727Z 2021-12-18T23:30:52.075672Z 2021-12-18T23:30:51.676096Z 2021-12-18T23:30:51.492655Z 2021-12-18T23:30:51.243567Z 2021-12-18T23:30:51.134838Z 2021-12-18T23:30:50.935923Z 2021-12-18T23:30:50.802222Z 2021-12-18T23:30:50.599356Z 2021-12-18T23:30:49.983656Z 2021-12-18T23:30:49.787823Z 2021-12-18T23:30:49.601888Z 2021-12-18T23:30:49.370951Z 2021-12-18T23:30:48.999227Z 2021-12-18T23:30:48.692748Z 2021-12-18T23:30:48.351643Z 2021-12-18T23:30:48.077664Z 2021-12-18T23:30:48.002305Z 2021-12-18T23:30:47.904957Z 2021-12-18T23:30:47.242008Z 2021-12-18T23:30:47.624893Z 2021-12-18T23:30:46.877176Z 2021-12-18T23:30:39.166659Z 2021-12-18T23:30:46.375355Z 2021-12-18T23:30:39.041162Z 2021-12-18T23:30:38.841700Z 2021-12-18T23:30:38.692233Z 2021-12-18T23:30:38.510470Z 2021-12-18T23:30:38.378445Z 2021-12-18T23:30:38.189773Z 2021-12-18T23:30:38.032853Z 2021-12-18T23:30:36.379065Z 2021-12-18T23:30:36.325533Z 2021-12-18T23:30:36.252423Z 2021-12-18T23:30:36.204226Z 2021-12-18T23:30:35.595177Z 2021-12-18T23:30:35.310176Z 2021-12-18T23:36:02.106537Z 2021-12-18T23:36:02.033633Z 2021-12-18T23:36:01.946673Z 2021-12-18T23:36:01.860789Z 2021-12-18T23:36:01.766842Z 2021-12-18T23:36:01.627797Z 2021-12-18T23:36:01.483181Z 2021-12-18T23:36:01.375588Z 2021-12-18T23:36:01.278533Z 2021-12-18T23:36:01.141811Z 2021-12-18T23:36:01.005964Z 2021-12-18T23:36:00.849746Z 2021-12-18T23:36:00.673414Z 2021-12-18T23:36:00.485708Z 2021-12-18T23:36:00.388313Z 2021-12-18T23:36:00.266569Z 2021-12-18T23:36:00.199660Z 2021-12-18T23:36:00.109737Z 2021-12-18T23:36:00.025102Z 2021-12-18T23:35:59.940586Z 2021-12-18T23:35:59.862659Z 2021-12-18T23:35:59.780981Z 2021-12-18T23:35:59.705603Z 2021-12-18T23:35:59.616167Z 2021-12-18T23:35:59.463363Z 2021-12-18T23:35:59.310007Z 2021-12-18T23:35:59.156969Z 2021-12-18T23:35:58.995870Z 2021-12-18T23:35:58.797843Z 2021-12-18T23:35:58.703668Z 2021-12-18T23:35:58.653440Z 2021-12-18T23:35:58.596887Z 2021-12-18T23:35:58.523671Z 2021-12-18T23:35:58.428643Z 2021-12-18T23:35:58.330125Z 2021-12-18T23:35:58.223708Z 2021-12-18T23:35:58.114897Z 2021-12-18T23:35:58.025656Z 2021-12-18T23:35:57.925708Z 2021-12-18T23:35:57.818680Z 2021-12-18T23:35:57.723774Z 2021-12-18T23:35:57.643750Z 2021-12-18T23:35:57.559589Z 2021-12-18T23:35:57.466790Z 2021-12-18T23:35:57.368634Z 2021-12-18T23:35:57.277582Z 2021-12-18T23:35:57.197601Z 2021-12-18T23:35:57.110581Z 2021-12-18T23:35:57.022855Z 2021-12-18T23:35:56.955637Z 2021-12-18T23:35:56.895621Z 2021-12-18T23:35:56.830472Z 2021-12-18T23:35:56.786485Z 2021-12-18T23:35:56.737743Z 2021-12-18T23:35:56.688441Z 2021-12-18T23:35:56.638962Z 2021-12-18T23:35:56.585479Z 2021-12-18T23:35:56.527663Z 2021-12-18T23:35:56.460948Z 2021-12-18T23:35:56.393592Z 2021-12-18T23:35:56.329666Z 2021-12-18T23:35:56.274480Z 2021-12-18T23:35:56.212464Z 2021-12-18T23:35:56.139105Z 2021-12-18T23:35:56.091692Z 2021-12-18T23:35:56.029561Z 2021-12-18T23:35:55.948609Z 2021-12-18T23:35:55.888490Z 2021-12-18T23:35:55.831190Z 2021-12-18T23:35:55.783424Z 2021-12-18T23:35:55.727550Z 2021-12-18T23:35:55.685652Z 2021-12-18T23:35:55.608627Z 2021-12-18T23:35:55.506661Z 2021-12-18T23:35:55.407936Z 2021-12-18T23:35:55.303783Z 2021-12-18T23:35:44.626519Z 2021-12-18T23:35:44.168633Z 2021-12-18T23:35:43.584235Z 2021-12-18T23:35:43.082395Z 2021-12-18T23:35:42.948804Z 2021-12-18T23:35:42.838719Z 2021-12-18T23:35:42.647363Z 2021-12-18T23:35:42.439889Z 2021-12-18T23:35:42.209394Z 2021-12-18T23:35:41.911155Z 2021-12-18T23:35:41.489484Z 2021-12-18T23:35:41.153616Z 2021-12-18T23:35:40.865218Z 2021-12-18T23:35:40.352625Z 2021-12-18T23:35:39.871633Z 2021-12-18T23:35:39.487322Z 2021-12-18T23:35:39.122049Z 2021-12-18T23:35:38.882264Z 2021-12-18T23:35:38.480665Z 2021-12-18T23:35:38.297523Z 2021-12-18T23:35:34.145508Z 2021-12-18T23:35:34.057391Z 2021-12-18T23:35:33.982287Z 2021-12-18T23:35:33.933324Z 2021-12-18T23:41:00.725936Z 2021-12-18T23:41:00.681890Z 2021-12-18T23:41:00.638395Z 2021-12-18T23:41:00.588656Z 2021-12-18T23:41:00.544184Z 2021-12-18T23:41:00.499858Z 2021-12-18T23:41:00.459985Z 2021-12-18T23:41:00.421470Z 2021-12-18T23:41:00.383501Z 2021-12-18T23:41:00.345756Z 2021-12-18T23:41:00.297038Z 2021-12-18T23:41:00.255021Z 2021-12-18T23:41:00.212623Z 2021-12-18T23:41:00.153964Z 2021-12-18T23:41:00.047798Z 2021-12-18T23:41:00.103413Z 2021-12-18T23:41:00.000463Z 2021-12-18T23:40:59.927768Z 2021-12-18T23:40:59.831854Z 2021-12-18T23:40:59.735647Z 2021-12-18T23:40:59.649730Z 2021-12-18T23:40:59.554469Z 2021-12-18T23:40:59.461473Z 2021-12-18T23:40:59.376609Z 2021-12-18T23:40:59.277524Z 2021-12-18T23:40:59.203606Z 2021-12-18T23:40:59.123647Z 2021-12-18T23:40:59.048621Z 2021-12-18T23:40:58.996621Z 2021-12-18T23:40:58.938680Z 2021-12-18T23:40:58.895583Z 2021-12-18T23:40:58.849738Z 2021-12-18T23:40:58.803976Z 2021-12-18T23:40:58.764673Z 2021-12-18T23:40:58.713181Z 2021-12-18T23:40:58.651515Z 2021-12-18T23:40:58.608904Z 2021-12-18T23:40:58.565836Z 2021-12-18T23:40:58.519319Z 2021-12-18T23:40:58.473362Z 2021-12-18T23:40:58.423498Z 2021-12-18T23:40:58.373008Z 2021-12-18T23:40:58.315630Z 2021-12-18T23:40:58.257463Z 2021-12-18T23:40:58.203856Z 2021-12-18T23:40:58.151495Z 2021-12-18T23:40:58.095444Z 2021-12-18T23:40:58.045463Z 2021-12-18T23:40:57.987450Z 2021-12-18T23:40:57.928590Z 2021-12-18T23:40:57.866431Z 2021-12-18T23:40:57.809397Z 2021-12-18T23:40:57.755386Z 2021-12-18T23:40:57.703708Z 2021-12-18T23:40:57.640528Z 2021-12-18T23:40:57.581487Z 2021-12-18T23:40:57.524653Z 2021-12-18T23:40:57.466454Z 2021-12-18T23:40:57.401582Z 2021-12-18T23:40:57.340496Z 2021-12-18T23:40:57.285556Z 2021-12-18T23:40:57.240529Z 2021-12-18T23:40:57.187683Z 2021-12-18T23:40:57.132807Z 2021-12-18T23:40:57.089827Z 2021-12-18T23:40:57.042168Z 2021-12-18T23:40:56.984590Z 2021-12-18T23:40:56.937352Z 2021-12-18T23:40:56.886508Z 2021-12-18T23:40:56.814470Z 2021-12-18T23:40:56.739687Z 2021-12-18T23:40:56.658505Z 2021-12-18T23:40:56.585118Z 2021-12-18T23:40:56.492745Z 2021-12-18T23:40:45.979648Z 2021-12-18T23:40:56.376027Z 2021-12-18T23:40:44.931517Z 2021-12-18T23:40:45.358245Z 2021-12-18T23:40:44.633433Z 2021-12-18T23:40:43.969180Z 2021-12-18T23:40:43.528814Z 2021-12-18T23:40:43.388874Z 2021-12-18T23:40:43.000899Z 2021-12-18T23:40:42.533247Z 2021-12-18T23:40:42.745392Z 2021-12-18T23:40:42.319354Z 2021-12-18T23:40:42.007475Z 2021-12-18T23:40:41.720435Z 2021-12-18T23:40:41.333896Z 2021-12-18T23:40:41.015600Z 2021-12-18T23:40:40.815420Z 2021-12-18T23:40:40.485812Z 2021-12-18T23:40:40.170677Z 2021-12-18T23:40:39.734319Z 2021-12-18T23:40:39.408173Z 2021-12-18T23:40:38.959164Z 2021-12-18T23:40:36.522388Z 2021-12-18T23:40:36.397837Z 2021-12-18T23:40:34.882843Z 2021-12-18T23:40:34.474757Z 2021-12-18T23:46:36.737932Z 2021-12-18T23:46:36.676874Z 2021-12-18T23:46:36.591281Z 2021-12-18T23:46:36.516438Z 2021-12-18T23:46:36.438365Z 2021-12-18T23:46:36.377702Z 2021-12-18T23:46:36.303410Z 2021-12-18T23:46:36.168663Z 2021-12-18T23:46:36.093662Z 2021-12-18T23:46:35.989647Z 2021-12-18T23:46:35.910973Z 2021-12-18T23:46:35.851825Z 2021-12-18T23:46:35.764681Z 2021-12-18T23:46:35.659470Z 2021-12-18T23:46:35.590678Z 2021-12-18T23:46:35.506414Z 2021-12-18T23:46:35.275886Z 2021-12-18T23:46:35.123609Z 2021-12-18T23:46:34.998746Z 2021-12-18T23:46:34.830562Z 2021-12-18T23:46:34.719542Z 2021-12-18T23:46:34.607472Z 2021-12-18T23:46:34.501987Z 2021-12-18T23:46:34.368624Z 2021-12-18T23:46:34.325340Z 2021-12-18T23:46:34.273429Z 2021-12-18T23:46:34.182772Z 2021-12-18T23:46:34.101194Z 2021-12-18T23:46:34.031396Z 2021-12-18T23:46:33.934075Z 2021-12-18T23:46:33.807646Z 2021-12-18T23:46:33.691285Z 2021-12-18T23:46:33.559761Z 2021-12-18T23:46:33.325562Z 2021-12-18T23:46:33.454681Z 2021-12-18T23:46:33.141810Z 2021-12-18T23:46:33.029682Z 2021-12-18T23:46:32.904686Z 2021-12-18T23:46:32.773727Z 2021-12-18T23:46:32.633906Z 2021-12-18T23:46:32.514468Z 2021-12-18T23:46:32.370777Z 2021-12-18T23:46:32.228464Z 2021-12-18T23:46:32.108486Z 2021-12-18T23:46:32.017795Z 2021-12-18T23:46:31.854671Z 2021-12-18T23:46:31.773119Z 2021-12-18T23:46:31.679797Z 2021-12-18T23:46:31.572770Z 2021-12-18T23:46:31.461467Z 2021-12-18T23:46:31.386603Z 2021-12-18T23:46:31.321450Z 2021-12-18T23:46:31.271774Z 2021-12-18T23:45:56.922459Z 2021-12-18T23:45:56.879806Z 2021-12-18T23:45:56.842524Z 2021-12-18T23:45:56.807895Z 2021-12-18T23:45:56.737795Z 2021-12-18T23:45:56.773271Z 2021-12-18T23:45:56.700681Z 2021-12-18T23:45:56.664350Z 2021-12-18T23:45:56.623416Z 2021-12-18T23:45:56.584794Z 2021-12-18T23:45:56.549638Z 2021-12-18T23:45:56.514448Z 2021-12-18T23:45:56.478419Z 2021-12-18T23:45:56.443164Z 2021-12-18T23:45:56.407638Z 2021-12-18T23:45:56.372467Z 2021-12-18T23:45:56.332471Z 2021-12-18T23:45:56.295542Z 2021-12-18T23:45:56.250639Z 2021-12-18T23:45:56.209212Z 2021-12-18T23:45:56.173802Z 2021-12-18T23:45:56.133327Z 2021-12-18T23:45:56.074002Z 2021-12-18T23:45:56.018204Z 2021-12-18T23:45:55.891290Z 2021-12-18T23:45:55.935977Z 2021-12-18T23:45:55.852177Z 2021-12-18T23:45:55.809714Z 2021-12-18T23:45:55.667440Z 2021-12-18T23:45:55.584893Z 2021-12-18T23:45:55.435011Z 2021-12-18T23:45:55.515891Z 2021-12-18T23:45:55.308092Z 2021-12-18T23:45:55.039951Z 2021-12-18T23:45:54.831324Z 2021-12-18T23:45:54.647734Z 2021-12-18T23:45:54.568135Z 2021-12-18T23:45:54.482340Z 2021-12-18T23:45:54.364814Z 2021-12-18T23:45:54.162478Z 2021-12-18T23:45:53.978600Z 2021-12-18T23:45:53.777831Z 2021-12-18T23:45:46.258549Z 2021-12-18T23:45:53.261672Z 2021-12-18T23:45:46.044090Z 2021-12-18T23:45:45.789346Z 2021-12-18T23:45:43.246195Z", + "modified_diff": 39.474313, + "started": "2021-12-18T23:25:46.448870Z 2021-12-18T23:25:45.631211Z 2021-12-18T23:25:45.621163Z 2021-12-18T23:25:44.716346Z 2021-12-18T23:25:44.788142Z 2021-12-18T23:25:44.152792Z 2021-12-18T23:25:44.052013Z 2021-12-18T23:25:43.506145Z 2021-12-18T23:25:43.496425Z 2021-12-18T23:25:42.931543Z 2021-12-18T23:25:42.678875Z 2021-12-18T23:25:42.528163Z 2021-12-18T23:25:27.405401Z 2021-12-18T23:25:27.592524Z 2021-12-18T23:25:26.555462Z 2021-12-18T23:25:26.833589Z 2021-12-18T23:25:26.413612Z 2021-12-18T23:25:26.022447Z 2021-12-18T23:25:25.327842Z 2021-12-18T23:25:25.096263Z 2021-12-18T23:25:25.086213Z 2021-12-18T23:25:24.572290Z 2021-12-18T23:25:24.156714Z 2021-12-18T23:25:24.039523Z 2021-12-18T23:25:23.403952Z 2021-12-18T23:25:23.355053Z 2021-12-18T23:25:22.872591Z 2021-12-18T23:25:19.367786Z 2021-12-18T23:25:19.214404Z 2021-12-18T23:25:19.123704Z 2021-12-18T23:25:18.546212Z 2021-12-18T23:25:17.193445Z 2021-12-18T23:25:16.917841Z 2021-12-18T23:25:16.195038Z 2021-12-18T23:25:16.090844Z 2021-12-18T23:25:14.576901Z 2021-12-18T23:26:13.011500Z 2021-12-18T23:26:12.137381Z 2021-12-18T23:26:12.075154Z 2021-12-18T23:26:11.354635Z 2021-12-18T23:26:10.758011Z 2021-12-18T23:26:10.561599Z 2021-12-18T23:26:10.225016Z 2021-12-18T23:26:09.756928Z 2021-12-18T23:26:09.531861Z 2021-12-18T23:26:09.273663Z 2021-12-18T23:26:08.829579Z 2021-12-18T23:26:08.204736Z 2021-12-18T23:26:08.568950Z 2021-12-18T23:26:07.532889Z 2021-12-18T23:26:07.594996Z 2021-12-18T23:26:06.919301Z 2021-12-18T23:26:07.095098Z 2021-12-18T23:26:06.375573Z 2021-12-18T23:26:06.467433Z 2021-12-18T23:26:05.863480Z 2021-12-18T23:26:06.114006Z 2021-12-18T23:26:05.604440Z 2021-12-18T23:26:05.782580Z 2021-12-18T23:26:05.592946Z 2021-12-18T23:26:05.376783Z 2021-12-18T23:26:05.111096Z 2021-12-18T23:26:04.908076Z 2021-12-18T23:25:59.933956Z 2021-12-18T23:25:59.649428Z 2021-12-18T23:25:59.310902Z 2021-12-18T23:25:58.942346Z 2021-12-18T23:25:58.743960Z 2021-12-18T23:25:58.340294Z 2021-12-18T23:25:57.833413Z 2021-12-18T23:25:57.378799Z 2021-12-18T23:25:57.482105Z 2021-12-18T23:25:56.757705Z 2021-12-18T23:25:56.358260Z 2021-12-18T23:25:55.876301Z 2021-12-18T23:26:04.671821Z 2021-12-18T23:25:55.458391Z 2021-12-18T23:25:54.976314Z 2021-12-18T23:25:54.600253Z 2021-12-18T23:25:54.503647Z 2021-12-18T23:25:54.499509Z 2021-12-18T23:25:53.690525Z 2021-12-18T23:25:53.892166Z 2021-12-18T23:25:52.712505Z 2021-12-18T23:25:52.622131Z 2021-12-18T23:25:52.034771Z 2021-12-18T23:25:51.596644Z 2021-12-18T23:25:51.177424Z 2021-12-18T23:25:50.924643Z 2021-12-18T23:25:50.565864Z 2021-12-18T23:25:50.354432Z 2021-12-18T23:25:49.756856Z 2021-12-18T23:25:49.527320Z 2021-12-18T23:25:48.824929Z 2021-12-18T23:25:48.590452Z 2021-12-18T23:25:48.144290Z 2021-12-18T23:25:47.669694Z 2021-12-18T23:25:47.141114Z 2021-12-18T23:25:46.964352Z 2021-12-18T23:25:46.425407Z 2021-12-18T23:31:38.170616Z 2021-12-18T23:31:37.935614Z 2021-12-18T23:31:37.408434Z 2021-12-18T23:31:36.885455Z 2021-12-18T23:31:36.136196Z 2021-12-18T23:31:35.480233Z 2021-12-18T23:31:34.696741Z 2021-12-18T23:31:34.295037Z 2021-12-18T23:31:33.854499Z 2021-12-18T23:31:33.535158Z 2021-12-18T23:31:32.579430Z 2021-12-18T23:31:32.317776Z 2021-12-18T23:31:31.698638Z 2021-12-18T23:31:31.631031Z 2021-12-18T23:31:30.507415Z 2021-12-18T23:31:30.071206Z 2021-12-18T23:31:29.732055Z 2021-12-18T23:31:29.581430Z 2021-12-18T23:31:28.950483Z 2021-12-18T23:31:28.944836Z 2021-12-18T23:31:28.523395Z 2021-12-18T23:31:28.095203Z 2021-12-18T23:31:27.606388Z 2021-12-18T23:31:27.339535Z 2021-12-18T23:31:26.601515Z 2021-12-18T23:31:26.281923Z 2021-12-18T23:31:25.688442Z 2021-12-18T23:31:25.222774Z 2021-12-18T23:31:24.826511Z 2021-12-18T23:31:24.338750Z 2021-12-18T23:31:23.711622Z 2021-12-18T23:31:23.100687Z 2021-12-18T23:31:22.700453Z 2021-12-18T23:31:22.620381Z 2021-12-18T23:31:22.029386Z 2021-12-18T23:31:21.576666Z 2021-12-18T23:31:21.048744Z 2021-12-18T23:31:20.854516Z 2021-12-18T23:31:20.530349Z 2021-12-18T23:31:20.326651Z 2021-12-18T23:31:19.877498Z 2021-12-18T23:31:19.698228Z 2021-12-18T23:31:19.097940Z 2021-12-18T23:31:18.614043Z 2021-12-18T23:31:18.234504Z 2021-12-18T23:31:17.818447Z 2021-12-18T23:31:17.307182Z 2021-12-18T23:31:16.947328Z 2021-12-18T23:31:16.660460Z 2021-12-18T23:31:16.411306Z 2021-12-18T23:31:15.975315Z 2021-12-18T23:31:15.665089Z 2021-12-18T23:31:06.677995Z 2021-12-18T23:31:06.369455Z 2021-12-18T23:31:15.469120Z 2021-12-18T23:31:06.245386Z 2021-12-18T23:31:05.285572Z 2021-12-18T23:31:05.773554Z 2021-12-18T23:31:04.936486Z 2021-12-18T23:31:04.484108Z 2021-12-18T23:31:04.393674Z 2021-12-18T23:31:03.860431Z 2021-12-18T23:31:03.302866Z 2021-12-18T23:31:03.216817Z 2021-12-18T23:31:03.192453Z 2021-12-18T23:31:03.270088Z 2021-12-18T23:31:03.181816Z 2021-12-18T23:31:01.965665Z 2021-12-18T23:31:01.502441Z 2021-12-18T23:31:01.578612Z 2021-12-18T23:31:00.728814Z 2021-12-18T23:31:00.816070Z 2021-12-18T23:31:00.175283Z 2021-12-18T23:30:59.696502Z 2021-12-18T23:30:59.327437Z 2021-12-18T23:30:58.960310Z 2021-12-18T23:30:58.664353Z 2021-12-18T23:30:58.519981Z 2021-12-18T23:30:58.061481Z 2021-12-18T23:30:57.832638Z 2021-12-18T23:30:57.307154Z 2021-12-18T23:30:56.758798Z 2021-12-18T23:30:56.348508Z 2021-12-18T23:30:56.677421Z 2021-12-18T23:30:56.274751Z 2021-12-18T23:30:43.245428Z 2021-12-18T23:30:55.986192Z 2021-12-18T23:30:42.527347Z 2021-12-18T23:30:42.070920Z 2021-12-18T23:30:41.981933Z 2021-12-18T23:30:41.221335Z 2021-12-18T23:30:40.744904Z 2021-12-18T23:30:40.396476Z 2021-12-18T23:30:40.023168Z 2021-12-18T23:30:37.579522Z 2021-12-18T23:30:37.605174Z 2021-12-18T23:30:36.977560Z 2021-12-18T23:30:36.754244Z 2021-12-18T23:30:35.798639Z 2021-12-18T23:30:35.495564Z 2021-12-18T23:36:37.395038Z 2021-12-18T23:36:36.971484Z 2021-12-18T23:36:35.884183Z 2021-12-18T23:36:36.417993Z 2021-12-18T23:36:35.341227Z 2021-12-18T23:36:35.008638Z 2021-12-18T23:36:34.317179Z 2021-12-18T23:36:33.819900Z 2021-12-18T23:36:33.518225Z 2021-12-18T23:36:33.640765Z 2021-12-18T23:36:32.806293Z 2021-12-18T23:36:32.868051Z 2021-12-18T23:36:32.484205Z 2021-12-18T23:36:32.288313Z 2021-12-18T23:36:31.747243Z 2021-12-18T23:36:31.538461Z 2021-12-18T23:36:31.130228Z 2021-12-18T23:36:30.790221Z 2021-12-18T23:36:30.283892Z 2021-12-18T23:36:30.427319Z 2021-12-18T23:36:29.228781Z 2021-12-18T23:36:28.926180Z 2021-12-18T23:36:28.224747Z 2021-12-18T23:36:27.946615Z 2021-12-18T23:36:27.018416Z 2021-12-18T23:36:26.518894Z 2021-12-18T23:36:25.741569Z 2021-12-18T23:36:25.688702Z 2021-12-18T23:36:24.198371Z 2021-12-18T23:36:23.619433Z 2021-12-18T23:36:23.223392Z 2021-12-18T23:36:24.140540Z 2021-12-18T23:36:22.927573Z 2021-12-18T23:36:22.813520Z 2021-12-18T23:36:21.658343Z 2021-12-18T23:36:21.026530Z 2021-12-18T23:36:20.567401Z 2021-12-18T23:36:19.936279Z 2021-12-18T23:36:19.270677Z 2021-12-18T23:36:19.027917Z 2021-12-18T23:36:18.644400Z 2021-12-18T23:36:18.397535Z 2021-12-18T23:36:17.767240Z 2021-12-18T23:36:17.296684Z 2021-12-18T23:36:16.662501Z 2021-12-18T23:36:16.151445Z 2021-12-18T23:36:15.929305Z 2021-12-18T23:36:15.399610Z 2021-12-18T23:36:14.416336Z 2021-12-18T23:36:14.041409Z 2021-12-18T23:36:13.467760Z 2021-12-18T23:36:13.016810Z 2021-12-18T23:36:12.168443Z 2021-12-18T23:36:11.755022Z 2021-12-18T23:36:11.705329Z 2021-12-18T23:36:11.091736Z 2021-12-18T23:36:10.910979Z 2021-12-18T23:36:10.896834Z 2021-12-18T23:36:10.057708Z 2021-12-18T23:36:09.539195Z 2021-12-18T23:36:09.062193Z 2021-12-18T23:36:08.255655Z 2021-12-18T23:36:07.461728Z 2021-12-18T23:36:07.077040Z 2021-12-18T23:36:07.073274Z 2021-12-18T23:36:06.577084Z 2021-12-18T23:36:05.877409Z 2021-12-18T23:36:05.896778Z 2021-12-18T23:36:05.192644Z 2021-12-18T23:36:05.108938Z 2021-12-18T23:36:03.943083Z 2021-12-18T23:36:03.778428Z 2021-12-18T23:36:03.382053Z 2021-12-18T23:36:03.162903Z 2021-12-18T23:36:02.785234Z 2021-12-18T23:36:02.635424Z 2021-12-18T23:35:52.709317Z 2021-12-18T23:35:52.190901Z 2021-12-18T23:35:51.847916Z 2021-12-18T23:35:51.273750Z 2021-12-18T23:35:50.915747Z 2021-12-18T23:35:50.683693Z 2021-12-18T23:35:50.343541Z 2021-12-18T23:35:50.111355Z 2021-12-18T23:35:49.876377Z 2021-12-18T23:35:49.543261Z 2021-12-18T23:35:49.265619Z 2021-12-18T23:35:48.999177Z 2021-12-18T23:35:48.681567Z 2021-12-18T23:35:48.696025Z 2021-12-18T23:35:48.289581Z 2021-12-18T23:35:47.917975Z 2021-12-18T23:35:47.593077Z 2021-12-18T23:35:47.172576Z 2021-12-18T23:35:46.229291Z 2021-12-18T23:35:46.200268Z 2021-12-18T23:35:36.597973Z 2021-12-18T23:35:35.786384Z 2021-12-18T23:35:35.586519Z 2021-12-18T23:35:34.577567Z 2021-12-18T23:41:36.460590Z 2021-12-18T23:41:35.711241Z 2021-12-18T23:41:35.058500Z 2021-12-18T23:41:34.355132Z 2021-12-18T23:41:33.967524Z 2021-12-18T23:41:33.485398Z 2021-12-18T23:41:33.391764Z 2021-12-18T23:41:32.744282Z 2021-12-18T23:41:32.127586Z 2021-12-18T23:41:31.361515Z 2021-12-18T23:41:30.937968Z 2021-12-18T23:41:30.456195Z 2021-12-18T23:41:30.177345Z 2021-12-18T23:41:29.880067Z 2021-12-18T23:41:28.656262Z 2021-12-18T23:41:29.353484Z 2021-12-18T23:41:28.580012Z 2021-12-18T23:41:26.454517Z 2021-12-18T23:41:26.077176Z 2021-12-18T23:41:25.565071Z 2021-12-18T23:41:25.201175Z 2021-12-18T23:41:25.023257Z 2021-12-18T23:41:25.022680Z 2021-12-18T23:41:23.759504Z 2021-12-18T23:41:23.520650Z 2021-12-18T23:41:23.052056Z 2021-12-18T23:41:23.201940Z 2021-12-18T23:41:21.760848Z 2021-12-18T23:41:21.676105Z 2021-12-18T23:41:20.462387Z 2021-12-18T23:41:19.995769Z 2021-12-18T23:41:19.278302Z 2021-12-18T23:41:18.467074Z 2021-12-18T23:41:17.830990Z 2021-12-18T23:41:17.950435Z 2021-12-18T23:41:16.951450Z 2021-12-18T23:41:16.847105Z 2021-12-18T23:41:15.506046Z 2021-12-18T23:41:15.494987Z 2021-12-18T23:41:15.048650Z 2021-12-18T23:41:15.288776Z 2021-12-18T23:41:14.622598Z 2021-12-18T23:41:14.201473Z 2021-12-18T23:41:13.478934Z 2021-12-18T23:41:13.362189Z 2021-12-18T23:41:12.602271Z 2021-12-18T23:41:12.809501Z 2021-12-18T23:41:11.657032Z 2021-12-18T23:41:11.770883Z 2021-12-18T23:41:10.975389Z 2021-12-18T23:41:11.114277Z 2021-12-18T23:41:10.127637Z 2021-12-18T23:41:09.784543Z 2021-12-18T23:41:09.205283Z 2021-12-18T23:41:08.935073Z 2021-12-18T23:41:08.160256Z 2021-12-18T23:41:07.883457Z 2021-12-18T23:41:07.333716Z 2021-12-18T23:41:07.949995Z 2021-12-18T23:41:06.360760Z 2021-12-18T23:41:06.093743Z 2021-12-18T23:41:05.245312Z 2021-12-18T23:41:04.961144Z 2021-12-18T23:41:04.406060Z 2021-12-18T23:41:03.899012Z 2021-12-18T23:41:03.782323Z 2021-12-18T23:41:03.308606Z 2021-12-18T23:41:02.970840Z 2021-12-18T23:41:02.638628Z 2021-12-18T23:41:02.427581Z 2021-12-18T23:41:02.218480Z 2021-12-18T23:41:01.845350Z 2021-12-18T23:41:01.743632Z 2021-12-18T23:41:01.529990Z 2021-12-18T23:40:53.489513Z 2021-12-18T23:41:01.246456Z 2021-12-18T23:40:52.689111Z 2021-12-18T23:40:53.167527Z 2021-12-18T23:40:52.134557Z 2021-12-18T23:40:51.733037Z 2021-12-18T23:40:51.380951Z 2021-12-18T23:40:50.978414Z 2021-12-18T23:40:50.816805Z 2021-12-18T23:40:50.156462Z 2021-12-18T23:40:50.381159Z 2021-12-18T23:40:49.820178Z 2021-12-18T23:40:49.839806Z 2021-12-18T23:40:49.521217Z 2021-12-18T23:40:49.213945Z 2021-12-18T23:40:49.099996Z 2021-12-18T23:40:48.752946Z 2021-12-18T23:40:48.627712Z 2021-12-18T23:40:48.354039Z 2021-12-18T23:40:47.988164Z 2021-12-18T23:40:47.259130Z 2021-12-18T23:40:47.615261Z 2021-12-18T23:40:38.285680Z 2021-12-18T23:40:37.826063Z 2021-12-18T23:40:35.248326Z 2021-12-18T23:40:34.690301Z 2021-12-18T23:47:03.853060Z 2021-12-18T23:47:03.665446Z 2021-12-18T23:47:02.898187Z 2021-12-18T23:47:02.859795Z 2021-12-18T23:47:02.259095Z 2021-12-18T23:47:02.080850Z 2021-12-18T23:47:01.671763Z 2021-12-18T23:47:01.380634Z 2021-12-18T23:47:00.869197Z 2021-12-18T23:47:00.581580Z 2021-12-18T23:47:00.271639Z 2021-12-18T23:46:59.579339Z 2021-12-18T23:46:59.023628Z 2021-12-18T23:46:58.615278Z 2021-12-18T23:46:58.154900Z 2021-12-18T23:46:57.570476Z 2021-12-18T23:46:56.924263Z 2021-12-18T23:46:56.457184Z 2021-12-18T23:46:55.304142Z 2021-12-18T23:46:54.816537Z 2021-12-18T23:46:53.920119Z 2021-12-18T23:46:53.762650Z 2021-12-18T23:46:52.873485Z 2021-12-18T23:46:52.707747Z 2021-12-18T23:46:52.130483Z 2021-12-18T23:46:51.174906Z 2021-12-18T23:46:50.997301Z 2021-12-18T23:46:50.142090Z 2021-12-18T23:46:49.306825Z 2021-12-18T23:46:48.842551Z 2021-12-18T23:46:48.184112Z 2021-12-18T23:46:48.544655Z 2021-12-18T23:46:47.409640Z 2021-12-18T23:46:46.757617Z 2021-12-18T23:46:48.036902Z 2021-12-18T23:46:48.084824Z 2021-12-18T23:46:45.173740Z 2021-12-18T23:46:45.069144Z 2021-12-18T23:46:43.774847Z 2021-12-18T23:46:44.283367Z 2021-12-18T23:46:42.906127Z 2021-12-18T23:46:41.632824Z 2021-12-18T23:46:41.345647Z 2021-12-18T23:46:40.764327Z 2021-12-18T23:46:39.865084Z 2021-12-18T23:46:39.908641Z 2021-12-18T23:46:39.356115Z 2021-12-18T23:46:39.008966Z 2021-12-18T23:46:38.270227Z 2021-12-18T23:46:38.157633Z 2021-12-18T23:46:37.758841Z 2021-12-18T23:46:37.483919Z 2021-12-18T23:46:37.328776Z 2021-12-18T23:46:11.417222Z 2021-12-18T23:46:10.937883Z 2021-12-18T23:46:11.255258Z 2021-12-18T23:46:10.853450Z 2021-12-18T23:46:09.499712Z 2021-12-18T23:46:09.296433Z 2021-12-18T23:46:08.626349Z 2021-12-18T23:46:08.199316Z 2021-12-18T23:46:07.753306Z 2021-12-18T23:46:07.354601Z 2021-12-18T23:46:07.259252Z 2021-12-18T23:46:06.630508Z 2021-12-18T23:46:06.293260Z 2021-12-18T23:46:06.042616Z 2021-12-18T23:46:06.081882Z 2021-12-18T23:46:05.501542Z 2021-12-18T23:46:05.023385Z 2021-12-18T23:46:04.787351Z 2021-12-18T23:46:04.431714Z 2021-12-18T23:46:03.952155Z 2021-12-18T23:46:03.675891Z 2021-12-18T23:46:03.348722Z 2021-12-18T23:46:03.047199Z 2021-12-18T23:46:02.645202Z 2021-12-18T23:46:02.086350Z 2021-12-18T23:46:02.272428Z 2021-12-18T23:46:01.569223Z 2021-12-18T23:46:00.580343Z 2021-12-18T23:45:59.921888Z 2021-12-18T23:45:59.569555Z 2021-12-18T23:45:59.157923Z 2021-12-18T23:45:59.370218Z 2021-12-18T23:45:58.823341Z 2021-12-18T23:45:58.722742Z 2021-12-18T23:45:58.495632Z 2021-12-18T23:45:58.227566Z 2021-12-18T23:45:57.981478Z 2021-12-18T23:45:57.816019Z 2021-12-18T23:45:57.622312Z 2021-12-18T23:45:57.532206Z 2021-12-18T23:45:57.383721Z 2021-12-18T23:45:57.303791Z 2021-12-18T23:45:48.118942Z 2021-12-18T23:45:57.165800Z 2021-12-18T23:45:47.705203Z 2021-12-18T23:45:47.116504Z 2021-12-18T23:45:43.558369Z", + "started_diff": 65.1984204, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 93, 91, 90, 89, 88, 87, 86, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 72, 71, 70, 69, 68, 67, 66, 64, 63, 62, 61, 60, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 153, 152, 150, 149, 147, 146, 145, 144, 143, 141, 140, 139, 136, 135, 134, 133, 132, 131, 129, 128, 127, 126, 124, 123, 122, 121, 120, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472]" + }, + "started": "2021-12-19T00:15:37.621571+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-21T03_04_04_43_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-21T03_04_04_43_0000-chatty.json new file mode 100644 index 00000000..5824a434 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-21T03_04_04_43_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-21T03_04_04_43_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-21T04:48:22.686151+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-api.json b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-api.json new file mode 100644 index 00000000..e2f229a8 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-api.json @@ -0,0 +1,1245 @@ +{ + "ended": "2021-12-21T23:45:55.793023+00:00", + "golden": "true", + "id": "run-2021-12-21T20_47_48_348_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2396.7833333333333, + "max": 26692.933333333334, + "mean": 5335.414059242389, + "median": 3466.6666666666665, + "min": 1362.7333333333333, + "non_zero_mean": 5335.414059242389, + "non_zero_median": 3466.6666666666665, + "percentile25": 2766.8333333333335, + "percentile75": 5163.616666666667, + "percentile90": 12148.053333333333, + "percentile99": 24002.07333333333, + "percentile999": 26041.860133333415, + "range": 25330.2, + "samples": 262, + "stdev": 4801.626220862205, + "sum": 1397878.4835215053 + } + }, + "cpu": { + "system": { + "iqr": 0.19783333333332392, + "max": 1.720666666666659, + "mean": 0.27114249419436887, + "median": 0.11533333333333076, + "min": 0.029333333333333184, + "non_zero_mean": 0.27114249419436887, + "non_zero_median": 0.11533333333333076, + "percentile25": 0.06133333333333818, + "percentile75": 0.2591666666666621, + "percentile90": 0.8438666666666641, + "percentile99": 1.6590066666666656, + "percentile999": 1.7077906666666616, + "range": 1.6913333333333258, + "samples": 262, + "stdev": 0.3797777252655939, + "sum": 71.03933347892463 + }, + "user": { + "iqr": 0.5519999999999641, + "max": 6.449333333333364, + "mean": 1.3594910984716917, + "median": 0.7690003778029777, + "min": 0.3913333333333715, + "non_zero_mean": 1.3594910984716917, + "non_zero_median": 0.7690003778029777, + "percentile25": 0.6018333333333545, + "percentile75": 1.1538333333333186, + "percentile90": 3.657133333333309, + "percentile99": 6.232033333333353, + "percentile999": 6.419231333333367, + "range": 6.057999999999993, + "samples": 262, + "stdev": 1.4353516604342273, + "sum": 356.18666779958323 + } + }, + "entropy_available_bits": { + "iqr": 2.166666666666667, + "max": 215.4, + "mean": 20.0870232251254, + "median": 1.7666666666666666, + "min": 0.13333333333333333, + "non_zero_mean": 20.0870232251254, + "non_zero_median": 1.7666666666666666, + "percentile25": 0.6666666666666666, + "percentile75": 2.8333333333333335, + "percentile90": 7.386666666666671, + "percentile99": 212.55933333333334, + "percentile999": 214.79100000000008, + "range": 215.26666666666668, + "samples": 262, + "stdev": 55.97611404540524, + "sum": 5262.800084982852 + }, + "memory": { + "used": { + "iqr": 396065792.0, + "max": 15524155392.0, + "mean": 6490984244.763359, + "median": 6094297088.0, + "min": 5660151808.0, + "non_zero_mean": 6490984244.763359, + "non_zero_median": 6094297088.0, + "percentile25": 5961857024.0, + "percentile75": 6357922816.0, + "percentile90": 7027672268.8, + "percentile99": 12429520609.279999, + "percentile999": 14996606189.568066, + "range": 9864003584.0, + "samples": 262, + "stdev": 1327896586.440616, + "sum": 1700637872128.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.7333333333333334, + "mean": 0.01628498727735369, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.4222222222222223, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.3119999999999891, + "percentile999": 1.7333333333333334, + "range": 1.7333333333333334, + "samples": 262, + "stdev": 0.15877170140389563, + "sum": 4.266666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 89.73333333333332, + "max": 1358.2666666666667, + "mean": 117.8824203712809, + "median": 25.4, + "min": 0.13333333333333333, + "non_zero_mean": 117.8824203712809, + "non_zero_median": 25.4, + "percentile25": 1.6, + "percentile75": 91.33333333333331, + "percentile90": 354.64000000000004, + "percentile99": 1210.142666666666, + "percentile999": 1342.5022666666687, + "range": 1358.1333333333332, + "samples": 262, + "stdev": 245.43836539100974, + "sum": 30885.19413727559 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 978.0333333333333, + "max": 3333.133333333333, + "mean": 1609.470737913486, + "median": 1495.9666666666667, + "min": 401.26666666666665, + "non_zero_mean": 1609.470737913486, + "non_zero_median": 1495.9666666666667, + "percentile25": 1145.35, + "percentile75": 2123.383333333333, + "percentile90": 2415.88, + "percentile99": 2926.901333333332, + "percentile999": 3250.6747333333437, + "range": 2931.866666666667, + "samples": 262, + "stdev": 601.8946211775933, + "sum": 421681.3333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.045166666666668936, + "max": 0.2953333333333319, + "mean": 0.08585496183206107, + "median": 0.07533333333333256, + "min": 0.01333333333333447, + "non_zero_mean": 0.08585496183206107, + "non_zero_median": 0.07533333333333256, + "percentile25": 0.05599999999999833, + "percentile75": 0.10116666666666727, + "percentile90": 0.15386666666666676, + "percentile99": 0.22134666666666783, + "percentile999": 0.280369333333334, + "range": 0.2819999999999974, + "samples": 262, + "stdev": 0.04575111002497161, + "sum": 22.494 + }, + "user": { + "iqr": 0.1306666666666686, + "max": 1.1533333333333342, + "mean": 0.20772010178117048, + "median": 0.20366666666666522, + "min": 0.01999999999999318, + "non_zero_mean": 0.20772010178117048, + "non_zero_median": 0.20366666666666522, + "percentile25": 0.13716666666666605, + "percentile75": 0.26783333333333464, + "percentile90": 0.32773333333333354, + "percentile99": 0.5097799999999963, + "percentile999": 1.0071733333333528, + "range": 1.133333333333341, + "samples": 262, + "stdev": 0.11375399219165164, + "sum": 54.422666666666686 + } + }, + "entropy_available_bits": { + "iqr": 1.7166666666666668, + "max": 212.13333333333333, + "mean": 11.280661577608143, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 22.734871794871793, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 1.7166666666666668, + "percentile90": 3.3333333333333335, + "percentile99": 211.6, + "percentile999": 212.13333333333333, + "range": 212.13333333333333, + "samples": 262, + "stdev": 45.80533624490848, + "sum": 2955.533333333335 + }, + "memory": { + "used": { + "iqr": 23429120.0, + "max": 1083396096.0, + "mean": 425000616.0610687, + "median": 383954944.0, + "min": 366395392.0, + "non_zero_mean": 425000616.0610687, + "non_zero_median": 383954944.0, + "percentile25": 377542656.0, + "percentile75": 400971776.0, + "percentile90": 530262425.6000001, + "percentile99": 960521052.1599998, + "percentile999": 1074594557.952001, + "range": 717000704.0, + "samples": 262, + "stdev": 121750222.58738053, + "sum": 111350161408.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 5870.9333333333325, + "max": 59607781.266666666, + "mean": 1346699.4386768448, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3835165.7927536233, + "non_zero_median": 13380.266666666666, + "percentile25": 0.0, + "percentile75": 5870.9333333333325, + "percentile90": 34734.08000000001, + "percentile99": 57839947.666666664, + "percentile999": 59515557.36906668, + "range": 59607781.266666666, + "samples": 262, + "stdev": 8739975.054626442, + "sum": 352835252.9333333 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 429.790048, + "mean": 2.8339537290076335, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 19.539365184210528, + "non_zero_median": 0.040191000000000004, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.003885100000000004, + "percentile99": 69.42109270999966, + "percentile999": 360.5664187870087, + "range": 429.790048, + "samples": 262, + "stdev": 29.0620200972657, + "sum": 742.4958770000002 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.40599025, + "max": 414.789772, + "mean": 25.833022729007634, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 55.026438658536584, + "non_zero_median": 0.877566, + "percentile25": 0.0, + "percentile75": 0.40599025, + "percentile90": 88.06640840000009, + "percentile99": 375.6608077699998, + "percentile999": 410.8780945300005, + "range": 414.789772, + "samples": 262, + "stdev": 77.68396732781183, + "sum": 6768.251955000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.8666666666666667, + "mean": 0.1351145038167939, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.723076923076923, + "non_zero_median": 2.7333333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.4259999999999993, + "percentile999": 3.7970666666666752, + "range": 3.8666666666666667, + "samples": 262, + "stdev": 0.6197379659832394, + "sum": 35.400000000000006 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 10000.733333333334, + "mean": 407.15776081424934, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8205.794871794871, + "non_zero_median": 8506.066666666668, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9965.351999999999, + "percentile999": 9999.532733333333, + "range": 10000.733333333334, + "samples": 262, + "stdev": 1826.7025473763663, + "sum": 106675.33333333334 + }, + "pg_stat_database_blks_hit": { + "iqr": 22820.35, + "max": 211082.26666666666, + "mean": 25822.08396946565, + "median": 26219.033333333333, + "min": 2372.733333333333, + "non_zero_mean": 25822.08396946565, + "non_zero_median": 26219.033333333333, + "percentile25": 10905.15, + "percentile75": 33725.5, + "percentile90": 45633.37333333334, + "percentile99": 67417.5399999996, + "percentile999": 198856.8004666682, + "range": 208709.53333333333, + "samples": 262, + "stdev": 20689.066049355413, + "sum": 6765385.999999996 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5833333333333333, + "max": 214.86666666666667, + "mean": 1.4071246819338423, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.8525963149078728, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6499999999999999, + "percentile90": 1.7333333333333334, + "percentile99": 4.411333333333331, + "percentile999": 160.5090666666735, + "range": 214.86666666666667, + "samples": 262, + "stdev": 13.272637718531437, + "sum": 368.66666666666634 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 262, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5833333333333333, + "max": 214.86666666666667, + "mean": 1.4071246819338423, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.8525963149078728, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6499999999999999, + "percentile90": 1.7333333333333334, + "percentile99": 4.411333333333331, + "percentile999": 160.5090666666735, + "range": 214.86666666666667, + "samples": 262, + "stdev": 13.272637718531437, + "sum": 368.66666666666634 + }, + "pg_stat_database_tup_deleted": { + "iqr": 3.0, + "max": 4005.866666666667, + "mean": 41.42315521628499, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 98.66242424242424, + "non_zero_median": 11.1, + "percentile25": 0.0, + "percentile75": 3.0, + "percentile90": 59.53333333333333, + "percentile99": 91.5039999999999, + "percentile999": 4005.2402666666667, + "range": 4005.866666666667, + "samples": 262, + "stdev": 348.985994524867, + "sum": 10852.866666666665 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11905.150000000001, + "max": 360042.06666666665, + "mean": 14684.468447837151, + "median": 14095.733333333334, + "min": 1249.2666666666667, + "non_zero_mean": 14684.468447837151, + "non_zero_median": 14095.733333333334, + "percentile25": 5496.716666666666, + "percentile75": 17401.86666666667, + "percentile90": 24856.12666666667, + "percentile99": 36404.679999999964, + "percentile999": 276275.3520666772, + "range": 358792.8, + "samples": 262, + "stdev": 23107.688365375176, + "sum": 3847330.7333333334 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.516666666666666, + "max": 8005.4, + "mean": 46.33638676844784, + "median": 2.5, + "min": 0.0, + "non_zero_mean": 55.94531490015361, + "non_zero_median": 3.8, + "percentile25": 1.0833333333333333, + "percentile75": 9.6, + "percentile90": 58.333333333333364, + "percentile99": 144.83199999999997, + "percentile999": 5954.566400000257, + "range": 8005.4, + "samples": 262, + "stdev": 494.71290068815284, + "sum": 12140.133333333333 + }, + "pg_stat_database_tup_returned": { + "iqr": 36283.73333333333, + "max": 510165.6666666667, + "mean": 45261.8272264631, + "median": 37048.26666666666, + "min": 3146.266666666667, + "non_zero_mean": 45261.8272264631, + "non_zero_median": 37048.26666666666, + "percentile25": 18818.9, + "percentile75": 55102.63333333333, + "percentile90": 91819.48000000001, + "percentile99": 158004.3179999999, + "percentile999": 489986.1906666692, + "range": 507019.4, + "samples": 262, + "stdev": 49845.77988926168, + "sum": 11858598.73333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.983333333333333, + "max": 71.8, + "mean": 10.386513994910942, + "median": 7.0, + "min": 0.2, + "non_zero_mean": 10.386513994910942, + "non_zero_median": 7.0, + "percentile25": 3.283333333333333, + "percentile75": 16.266666666666666, + "percentile90": 23.30666666666667, + "percentile99": 42.88866666666662, + "percentile999": 68.96380000000035, + "range": 71.6, + "samples": 262, + "stdev": 10.199476842505604, + "sum": 2721.266666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 125.45, + "max": 713.6666666666666, + "mean": 123.2267175572519, + "median": 86.66666666666667, + "min": 3.8666666666666667, + "non_zero_mean": 123.2267175572519, + "non_zero_median": 86.66666666666667, + "percentile25": 39.55, + "percentile75": 165.0, + "percentile90": 264.94666666666666, + "percentile99": 573.993333333333, + "percentile999": 706.5500666666675, + "range": 709.8, + "samples": 262, + "stdev": 125.51007381018879, + "sum": 32285.40000000001 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.1267175572519084, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.1267175572519084, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 262, + "stdev": 0.14527728349201977, + "sum": 33.2 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.8666666666666667, + "mean": 0.0071246819338422395, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.8666666666666667, + "non_zero_median": 1.8666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 1.379466666666728, + "range": 1.8666666666666667, + "samples": 262, + "stdev": 0.11532305180017356, + "sum": 1.8666666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 27.450000000000003, + "max": 168.8, + "mean": 39.17150127226463, + "median": 32.46666666666667, + "min": 2.0666666666666664, + "non_zero_mean": 39.17150127226463, + "non_zero_median": 32.46666666666667, + "percentile25": 22.483333333333334, + "percentile75": 49.93333333333334, + "percentile90": 71.81333333333333, + "percentile99": 116.33399999999995, + "percentile999": 157.17680000000146, + "range": 166.73333333333335, + "samples": 262, + "stdev": 24.612053075955767, + "sum": 10262.933333333332 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2021-12-21T22:40:33.276564+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2021-12-21T20_47_48_348_0000", + "ended": "2021-12-21T23:45:50.472743+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.011075825749912838, + "iterations": 1, + "max": 0.5626477739997426, + "mean": 0.5385043055999631, + "median": 0.5382009690001723, + "min": 0.520477772999584, + "rounds": 15, + "stddev": 0.012248211426842974 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.016028055999640856, + "iterations": 1, + "max": 0.5999861809996219, + "mean": 0.5485649628665973, + "median": 0.5437799740002447, + "min": 0.5209131739993609, + "rounds": 15, + "stddev": 0.02038511979638654 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.007804882999153051, + "iterations": 1, + "max": 0.41791532000024745, + "mean": 0.40851292510005804, + "median": 0.40676296400033607, + "min": 0.40254384800027765, + "rounds": 10, + "stddev": 0.005328358283665377 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.011156743000356073, + "iterations": 1, + "max": 0.32210774999930436, + "mean": 0.2622011757600376, + "median": 0.2613773845000651, + "min": 0.24466002000008302, + "rounds": 100, + "stddev": 0.010322676928296779 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.012858771499395516, + "iterations": 1, + "max": 0.5389251090000471, + "mean": 0.4717402070800017, + "median": 0.46890491949989155, + "min": 0.4489646400006677, + "rounds": 100, + "stddev": 0.012301537931081135 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.11540949900063424, + "iterations": 1, + "max": 1.443771446000028, + "mean": 1.3505298381001922, + "median": 1.3280388825005502, + "min": 1.2874455009996382, + "rounds": 10, + "stddev": 0.0635897992491359 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.00286166299974866, + "iterations": 1, + "max": 0.081914260999838, + "mean": 0.07667918707147692, + "median": 0.07636573550007597, + "min": 0.0729803450003601, + "rounds": 14, + "stddev": 0.002394389957424091 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0022905340001670993, + "iterations": 1, + "max": 0.07844212100008008, + "mean": 0.07362843521423201, + "median": 0.07295907050001915, + "min": 0.07158202899972821, + "rounds": 14, + "stddev": 0.0019122995029196945 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.015050597000026755, + "iterations": 1, + "max": 0.1321633790003034, + "mean": 0.10286455945466018, + "median": 0.09671489399988786, + "min": 0.0930372789998728, + "rounds": 11, + "stddev": 0.012893283600876548 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.0023424210003213375, + "iterations": 1, + "max": 0.11370831499971246, + "mean": 0.11161071589995117, + "median": 0.11203376699995715, + "min": 0.10662025600049674, + "rounds": 10, + "stddev": 0.002087726363732624 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.01421641599972645, + "iterations": 1, + "max": 0.305494125000223, + "mean": 0.2725565847001235, + "median": 0.2707402275000277, + "min": 0.2549043389999497, + "rounds": 10, + "stddev": 0.01425328698966247 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.005324020000443852, + "iterations": 1, + "max": 0.11826931100040383, + "mean": 0.11322365219994027, + "median": 0.11331417549990874, + "min": 0.1090203539997674, + "rounds": 10, + "stddev": 0.003026643669217413 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0010659889994713012, + "iterations": 1, + "max": 0.07829942999978812, + "mean": 0.07504334528558891, + "median": 0.07493299099996875, + "min": 0.07265714899949671, + "rounds": 14, + "stddev": 0.0014432918012851545 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.027045605999774125, + "iterations": 1, + "max": 0.6580416389997481, + "mean": 0.6263309004998519, + "median": 0.6293805264999719, + "min": 0.593098184000155, + "rounds": 10, + "stddev": 0.021037150894584113 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.004618563999656544, + "iterations": 1, + "max": 0.0958335220002482, + "mean": 0.09115203990925445, + "median": 0.09107627600042179, + "min": 0.08710452400009672, + "rounds": 11, + "stddev": 0.0029052519159317044 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.0013274410002850345, + "iterations": 1, + "max": 0.10189257700039889, + "mean": 0.09998078580010769, + "median": 0.1000510949997988, + "min": 0.09842908400059969, + "rounds": 10, + "stddev": 0.0010366459862308519 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.004594947999976284, + "iterations": 1, + "max": 0.18192370799988566, + "mean": 0.17533464150001238, + "median": 0.1754620309998245, + "min": 0.17057740299969737, + "rounds": 10, + "stddev": 0.003449083320826566 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.0019550520000848337, + "iterations": 1, + "max": 0.07630042500022682, + "mean": 0.07321134771427751, + "median": 0.07315223599971432, + "min": 0.07115518099999463, + "rounds": 14, + "stddev": 0.001440770119984548 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.0018213950006611412, + "iterations": 1, + "max": 0.08137051100038661, + "mean": 0.07791979664287803, + "median": 0.07850433849989713, + "min": 0.07378097800028627, + "rounds": 14, + "stddev": 0.0020553208806010216 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.0048065490009321366, + "iterations": 1, + "max": 0.4855248730000312, + "mean": 0.3868996610999602, + "median": 0.3768150439996134, + "min": 0.3678788219995113, + "rounds": 10, + "stddev": 0.03486121284776226 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.010374151999712922, + "iterations": 1, + "max": 0.38105259600069985, + "mean": 0.3738158178001868, + "median": 0.37487792900037675, + "min": 0.3653574829995705, + "rounds": 10, + "stddev": 0.005667025652908118 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.003887178500463051, + "iterations": 1, + "max": 0.10382812899933924, + "mean": 0.095399100908997, + "median": 0.0948137229997883, + "min": 0.09122921600010159, + "rounds": 11, + "stddev": 0.0034299449573608485 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.002540775500392556, + "iterations": 1, + "max": 0.09062735900079133, + "mean": 0.08738305609089862, + "median": 0.08666203200027667, + "min": 0.08526446999985637, + "rounds": 11, + "stddev": 0.0017429939716485415 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.0046154470001056325, + "iterations": 1, + "max": 0.2904549490003774, + "mean": 0.1924598920999415, + "median": 0.18201401800024541, + "min": 0.17760408499998448, + "rounds": 10, + "stddev": 0.03457323832554105 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0022092552505910135, + "iterations": 1, + "max": 0.07448357199973543, + "mean": 0.07270634606678263, + "median": 0.07265368299977126, + "min": 0.0701554320003197, + "rounds": 15, + "stddev": 0.0013442908279408935 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.011286776999440917, + "iterations": 1, + "max": 0.21240008999939164, + "mean": 0.20485849120013883, + "median": 0.2049878205002642, + "min": 0.1973177370000485, + "rounds": 10, + "stddev": 0.00614082307905195 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0028757780003161315, + "iterations": 1, + "max": 0.08189245500034303, + "mean": 0.07980204907695364, + "median": 0.07938052899953618, + "min": 0.07773011700010102, + "rounds": 13, + "stddev": 0.0014600378150720435 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.00621027499983029, + "iterations": 1, + "max": 0.36632764199930534, + "mean": 0.35755996499983667, + "median": 0.3568762864997552, + "min": 0.34850003799965634, + "rounds": 10, + "stddev": 0.004964296415537209 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.006431357000110438, + "iterations": 1, + "max": 0.23900132799917628, + "mean": 0.2316520006000246, + "median": 0.23284571249996588, + "min": 0.22491401699971902, + "rounds": 10, + "stddev": 0.004631001867101516 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.0071395589993699105, + "iterations": 1, + "max": 0.16923851600040507, + "mean": 0.16197962900014318, + "median": 0.16185065199988458, + "min": 0.15468196300025738, + "rounds": 10, + "stddev": 0.0046116783863360804 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.11392663399965386, + "iterations": 1, + "max": 0.23417462999987038, + "mean": 0.14233893200007514, + "median": 0.1068640750004306, + "min": 0.10527746099978685, + "rounds": 10, + "stddev": 0.05758572292338774 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.0012461899996196735, + "iterations": 1, + "max": 0.08766485300020577, + "mean": 0.07273588164298417, + "median": 0.07154019249992416, + "min": 0.06917062299999088, + "rounds": 14, + "stddev": 0.004599715858509881 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.005474173999573395, + "iterations": 1, + "max": 0.10439472300004127, + "mean": 0.0980031372727436, + "median": 0.09753863100013405, + "min": 0.09212286400088487, + "rounds": 11, + "stddev": 0.0037865635013875475 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0017981355003939825, + "iterations": 1, + "max": 0.08132212399959826, + "mean": 0.07605711599990173, + "median": 0.07554122999954416, + "min": 0.0740118789999542, + "rounds": 13, + "stddev": 0.0020038736168016716 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.0073242749995188206, + "iterations": 1, + "max": 0.1382588129999931, + "mean": 0.12799264440009211, + "median": 0.12737337549970107, + "min": 0.12029433699990477, + "rounds": 10, + "stddev": 0.005802119076254531 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0024480250003762194, + "iterations": 1, + "max": 0.08980717999929766, + "mean": 0.07577205221436348, + "median": 0.07497039800045968, + "min": 0.07238875400071265, + "rounds": 14, + "stddev": 0.00433154347387999 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.1328050030006125, + "iterations": 1, + "max": 0.8455440859997907, + "mean": 0.6632900354998128, + "median": 0.6197597054997459, + "min": 0.5993969800001651, + "rounds": 10, + "stddev": 0.08661492552730084 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.020715362999908393, + "iterations": 1, + "max": 0.6436462960000426, + "mean": 0.6200470429999768, + "median": 0.6156931155001075, + "min": 0.6009914129999743, + "rounds": 10, + "stddev": 0.01383963109729659 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.008250799999586889, + "iterations": 1, + "max": 0.2924955050002609, + "mean": 0.27974699930000496, + "median": 0.2774124625002514, + "min": 0.27045309000004636, + "rounds": 10, + "stddev": 0.0074070205818191764 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.00189738200060674, + "iterations": 1, + "max": 0.08147814499989181, + "mean": 0.07483016892852642, + "median": 0.07383568649993322, + "min": 0.0724408370006131, + "rounds": 14, + "stddev": 0.0028181687292845293 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0028643339996961004, + "iterations": 1, + "max": 0.07607551300043269, + "mean": 0.0737302483077236, + "median": 0.07294279400048254, + "min": 0.07175094499962142, + "rounds": 13, + "stddev": 0.0016287528637653904 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.001126403999478498, + "iterations": 1, + "max": 0.08584217600036936, + "mean": 0.07646554421411825, + "median": 0.07596875799981717, + "min": 0.07306208299996797, + "rounds": 14, + "stddev": 0.003004966232836484 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.002737410000008822, + "iterations": 1, + "max": 0.08716070699938427, + "mean": 0.07862334685699222, + "median": 0.07774737749969063, + "min": 0.0755342729999029, + "rounds": 14, + "stddev": 0.0030637601576532377 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.02102599200043187, + "iterations": 1, + "max": 0.30707604799954424, + "mean": 0.2921826450997287, + "median": 0.2948551654999392, + "min": 0.27136688599966874, + "rounds": 10, + "stddev": 0.011512808735352735 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.003695485999742232, + "iterations": 1, + "max": 0.23936173399943073, + "mean": 0.23004650510010832, + "median": 0.2295173910001722, + "min": 0.22456928100018558, + "rounds": 10, + "stddev": 0.004164119340210332 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.04114850199948705, + "iterations": 1, + "max": 1.0285442549993604, + "mean": 0.9155502666997564, + "median": 0.9275617179996516, + "min": 0.6633139219993609, + "rounds": 10, + "stddev": 0.09542206641099141 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.13597343000037654, + "iterations": 1, + "max": 9.030154773000504, + "mean": 8.77099110980007, + "median": 8.742626829500296, + "min": 8.611758689000453, + "rounds": 10, + "stddev": 0.12229449069313188 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.04363820199978363, + "iterations": 1, + "max": 1.0638710510002056, + "mean": 1.0318773029998738, + "median": 1.0396483784998054, + "min": 0.9795715199998085, + "rounds": 10, + "stddev": 0.02815124656033659 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.19900406200031284, + "iterations": 1, + "max": 9.295130618999792, + "mean": 9.04109478979999, + "median": 9.014066430499952, + "min": 8.86837389700031, + "rounds": 10, + "stddev": 0.14510462602321525 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.4493342420000772, + "iterations": 1, + "max": 1.603459328000099, + "mean": 1.3092676836002284, + "median": 1.2790355345000535, + "min": 1.0125221450007302, + "rounds": 10, + "stddev": 0.23999428038396914 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.008493116999488848, + "iterations": 1, + "max": 0.417705167999884, + "mean": 0.4060896399001649, + "median": 0.40895552649999445, + "min": 0.3809513060004974, + "rounds": 10, + "stddev": 0.010746593150263004 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.4102725319989986, + "iterations": 1, + "max": 11.868923381000059, + "mean": 11.561304166899754, + "median": 11.510553958499258, + "min": 11.122728682999877, + "rounds": 10, + "stddev": 0.26621441371002513 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.012215587999889976, + "iterations": 1, + "max": 0.44113677499990445, + "mean": 0.4153299251000135, + "median": 0.41449175300022034, + "min": 0.40063278900015575, + "rounds": 10, + "stddev": 0.011020263124645715 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.17320140400079254, + "iterations": 1, + "max": 0.9397948429996177, + "mean": 0.5917038525999487, + "median": 0.5432238104995122, + "min": 0.4267604950000532, + "rounds": 10, + "stddev": 0.16610545058805024 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.17057061599916779, + "iterations": 1, + "max": 12.002585665000879, + "mean": 11.709580563000419, + "median": 11.715574812500563, + "min": 11.376416852999682, + "rounds": 10, + "stddev": 0.17104780597347172 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.11519909799972083, + "iterations": 1, + "max": 7.887904725999761, + "mean": 7.7569937747998665, + "median": 7.743755450499975, + "min": 7.672468765999838, + "rounds": 10, + "stddev": 0.07048491626922461 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 264.22419869199985, + "mean": 264.22419869199985, + "median": 264.22419869199985, + "min": 264.22419869199985, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 390.2353752509989, + "mean": 390.2353752509989, + "median": 390.2353752509989, + "min": 390.2353752509989, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.006549604000610998, + "iterations": 1, + "max": 0.30001978800009965, + "mean": 0.26568447199988443, + "median": 0.26283559300009074, + "min": 0.25201559100059967, + "rounds": 10, + "stddev": 0.013150966543506502 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.014074949999667297, + "iterations": 1, + "max": 0.6645138200001384, + "mean": 0.6454236549000598, + "median": 0.64413456650027, + "min": 0.6294198309997228, + "rounds": 10, + "stddev": 0.009744361665594984 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.08576177100076166, + "iterations": 1, + "max": 9.76586836000024, + "mean": 9.606480757400096, + "median": 9.633922902500672, + "min": 9.245557325000846, + "rounds": 10, + "stddev": 0.14282830582250597 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.5068763869994655, + "iterations": 1, + "max": 3.5159325899994656, + "mean": 3.0472371905999354, + "median": 3.034066309499849, + "min": 2.263711274999878, + "rounds": 10, + "stddev": 0.3889131225360121 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.38020705899998575, + "iterations": 1, + "max": 3.5350661590000527, + "mean": 2.1409433992000233, + "median": 1.9938987360001192, + "min": 1.5111683949999133, + "rounds": 10, + "stddev": 0.540934146906354 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 1.5142003030014166, + "iterations": 1, + "max": 26.285720593001315, + "mean": 25.073456451799938, + "median": 25.003553233499588, + "min": 24.105982274999405, + "rounds": 10, + "stddev": 0.8467296850164294 + } + }, + "started": "2021-12-21T22:29:57.079760+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-chatty.json new file mode 100644 index 00000000..12ace68e --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2021-12-21T21:44:26.681197+00:00", + "golden": "true", + "id": "run-2021-12-21T20_47_48_348_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 13329.533333333333, + "max": 59214.2, + "mean": 32021.450445316637, + "median": 37915.6, + "min": 1726.8666666666668, + "non_zero_mean": 32021.450445316637, + "non_zero_median": 37915.6, + "percentile25": 26083.75, + "percentile75": 39413.28333333333, + "percentile90": 40243.86, + "percentile99": 47907.84199999996, + "percentile999": 58083.56420000008, + "range": 57487.33333333333, + "samples": 64, + "stdev": 11985.043602353555, + "sum": 2049372.828500265 + } + }, + "cpu": { + "system": { + "iqr": 0.26650000000000174, + "max": 1.0846666666666667, + "mean": 0.8647187354871758, + "median": 1.0066666666666653, + "min": 0.04133333333333363, + "non_zero_mean": 0.8647187354871758, + "non_zero_median": 1.0066666666666653, + "percentile25": 0.7686666666666662, + "percentile75": 1.035166666666668, + "percentile90": 1.0502666666666662, + "percentile99": 1.0720666666666678, + "percentile999": 1.0834066666666669, + "range": 1.043333333333333, + "samples": 64, + "stdev": 0.28148051310638594, + "sum": 55.341999071179224 + }, + "user": { + "iqr": 2.1626666666666656, + "max": 6.545333333333336, + "mean": 5.173656227511713, + "median": 6.285666666666674, + "min": 0.37133333333334423, + "non_zero_mean": 5.173656227511713, + "non_zero_median": 6.285666666666674, + "percentile25": 4.22433333333334, + "percentile75": 6.387000000000006, + "percentile90": 6.4310666666666645, + "percentile99": 6.499553333333331, + "percentile999": 6.540755333333336, + "range": 6.173999999999992, + "samples": 64, + "stdev": 1.863792834629285, + "sum": 331.1139985607497 + } + }, + "entropy_available_bits": { + "iqr": 8.416666666666666, + "max": 217.73333333333335, + "mean": 23.71874854206018, + "median": 7.033333333333333, + "min": 0.0, + "non_zero_mean": 32.297870355145776, + "non_zero_median": 7.733333333333333, + "percentile25": 0.0, + "percentile75": 8.416666666666666, + "percentile90": 18.846666666666675, + "percentile99": 217.01933333333332, + "percentile999": 217.66193333333337, + "range": 217.73333333333335, + "samples": 64, + "stdev": 58.07672095112858, + "sum": 1517.9999066918515 + }, + "memory": { + "used": { + "iqr": 1448940544.0, + "max": 7451996160.0, + "mean": 6368935616.0, + "median": 6799097856.0, + "min": 3428880384.0, + "non_zero_mean": 6368935616.0, + "non_zero_median": 6799097856.0, + "percentile25": 5629373440.0, + "percentile75": 7078313984.0, + "percentile90": 7203471769.6, + "percentile99": 7391378104.32, + "percentile999": 7445934354.432, + "range": 4023115776.0, + "samples": 64, + "stdev": 921606904.6452005, + "sum": 407611879424.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.0666666666666667, + "mean": 0.03958333333333333, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6333333333333333, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.0666666666666667, + "percentile999": 1.0666666666666667, + "range": 1.0666666666666667, + "samples": 64, + "stdev": 0.189576247033096, + "sum": 2.533333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 2081.0666666666675, + "max": 4815.266666666666, + "mean": 3565.422243981784, + "median": 4532.5, + "min": 7.066666666666666, + "non_zero_mean": 3565.422243981784, + "non_zero_median": 4532.5, + "percentile25": 2587.883333333333, + "percentile75": 4668.950000000001, + "percentile90": 4775.073333333334, + "percentile99": 4810.394666666666, + "percentile999": 4814.779466666667, + "range": 4808.2, + "samples": 64, + "stdev": 1536.027316264337, + "sum": 228187.02361483412 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 220.78333333333353, + "max": 2262.2, + "mean": 1618.490625, + "median": 1684.6, + "min": 555.0666666666667, + "non_zero_mean": 1618.490625, + "non_zero_median": 1684.6, + "percentile25": 1558.8333333333333, + "percentile75": 1779.6166666666668, + "percentile90": 1923.9266666666667, + "percentile99": 2171.1439999999993, + "percentile999": 2253.0944000000004, + "range": 1707.1333333333332, + "samples": 64, + "stdev": 333.7958195549142, + "sum": 103583.40000000002 + } + }, + "cpu": { + "system": { + "iqr": 0.031833333333333796, + "max": 0.23733333333333348, + "mean": 0.1445625, + "median": 0.15466666666666645, + "min": 0.02133333333333347, + "non_zero_mean": 0.1445625, + "non_zero_median": 0.15466666666666645, + "percentile25": 0.13466666666666677, + "percentile75": 0.16650000000000056, + "percentile90": 0.18846666666666656, + "percentile99": 0.23103333333333323, + "percentile999": 0.23670333333333352, + "range": 0.216, + "samples": 64, + "stdev": 0.04610305437359823, + "sum": 9.251999999999997 + }, + "user": { + "iqr": 0.038666666666667265, + "max": 0.5486666666666666, + "mean": 0.21932291666666665, + "median": 0.18033333333333418, + "min": 0.029999999999999714, + "non_zero_mean": 0.21932291666666665, + "non_zero_median": 0.18033333333333418, + "percentile25": 0.17049999999999935, + "percentile75": 0.2091666666666666, + "percentile90": 0.49826666666666775, + "percentile99": 0.5465666666666671, + "percentile999": 0.5484566666666667, + "range": 0.5186666666666669, + "samples": 64, + "stdev": 0.13372864636938642, + "sum": 14.036666666666669 + } + }, + "entropy_available_bits": { + "iqr": 1.5666666666666669, + "max": 186.73333333333332, + "mean": 9.938541666666666, + "median": 2.6, + "min": 0.3333333333333333, + "non_zero_mean": 9.938541666666666, + "non_zero_median": 2.6, + "percentile25": 1.4, + "percentile75": 2.966666666666667, + "percentile90": 4.566666666666667, + "percentile99": 176.1913333333333, + "percentile999": 185.6791333333334, + "range": 186.39999999999998, + "samples": 64, + "stdev": 34.78224182453437, + "sum": 636.0666666666665 + }, + "memory": { + "used": { + "iqr": 28956672.0, + "max": 473432064.0, + "mean": 443512000.0, + "median": 456292352.0, + "min": 363995136.0, + "non_zero_mean": 443512000.0, + "non_zero_median": 456292352.0, + "percentile25": 433280000.0, + "percentile75": 462236672.0, + "percentile90": 469123891.2, + "percentile99": 472709529.6, + "percentile999": 473359810.56, + "range": 109436928.0, + "samples": 64, + "stdev": 30172260.161224045, + "sum": 28384768000.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1458995.2, + "max": 3382203.7333333334, + "mean": 823330.1333333333, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2107725.141333333, + "non_zero_median": 2208563.2, + "percentile25": 0.0, + "percentile75": 1458995.2, + "percentile90": 3136826.0266666664, + "percentile99": 3359839.5733333332, + "percentile999": 3379967.3173333337, + "range": 3382203.7333333334, + "samples": 64, + "stdev": 1210541.8595639006, + "sum": 52693128.53333333 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.00025175, + "max": 0.025602, + "mean": 0.0030250625, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.01210025, + "non_zero_median": 0.0113475, + "percentile25": 0.0, + "percentile75": 0.00025175, + "percentile90": 0.013701100000000008, + "percentile99": 0.02272352999999999, + "percentile999": 0.02531415300000002, + "range": 0.025602, + "samples": 64, + "stdev": 0.006375339684383087, + "sum": 0.193604 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 1.154912, + "mean": 0.027090390625, + "median": 0.0, + "min": -0.000406, + "non_zero_mean": 0.1733785, + "non_zero_median": 0.0536685, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.009830700000000006, + "percentile99": 0.5722350799999976, + "percentile999": 1.0966443080000043, + "range": 1.1553179999999998, + "samples": 64, + "stdev": 0.1475001182731425, + "sum": 1.7337850000000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 6.4, + "mean": 0.15416666666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 4.933333333333334, + "non_zero_median": 4.933333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.5519999999999925, + "percentile999": 6.2152000000000145, + "range": 6.4, + "samples": 64, + "stdev": 0.9037546607196484, + "sum": 9.866666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9989.066666666668, + "mean": 312.125, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9988.0, + "non_zero_median": 9988.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9987.722666666667, + "percentile999": 9988.932266666667, + "range": 9989.066666666668, + "samples": 64, + "stdev": 1751.576529217959, + "sum": 19976.0 + }, + "pg_stat_database_blks_hit": { + "iqr": 2774.333333333332, + "max": 37898.0, + "mean": 21781.373958333334, + "median": 22851.63333333333, + "min": 4019.266666666667, + "non_zero_mean": 21781.373958333334, + "non_zero_median": 22851.63333333333, + "percentile25": 21077.883333333335, + "percentile75": 23852.216666666667, + "percentile90": 25294.093333333334, + "percentile99": 37738.778, + "percentile999": 37882.0778, + "range": 33878.73333333333, + "samples": 64, + "stdev": 5936.259033536529, + "sum": 1394007.9333333338 + }, + "pg_stat_database_blks_read": { + "iqr": 68.95, + "max": 156.8, + "mean": 100.45104166666667, + "median": 130.13333333333333, + "min": 0.0, + "non_zero_mean": 102.04550264550265, + "non_zero_median": 130.53333333333333, + "percentile25": 66.7, + "percentile75": 135.65, + "percentile90": 138.52, + "percentile99": 152.26399999999998, + "percentile999": 156.34640000000005, + "range": 156.8, + "samples": 64, + "stdev": 49.66443124430306, + "sum": 6428.866666666667 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 68.95, + "max": 156.8, + "mean": 100.45104166666667, + "median": 130.13333333333333, + "min": 0.0, + "non_zero_mean": 102.04550264550265, + "non_zero_median": 130.53333333333333, + "percentile25": 66.7, + "percentile75": 135.65, + "percentile90": 138.52, + "percentile99": 152.26399999999998, + "percentile999": 156.34640000000005, + "range": 156.8, + "samples": 64, + "stdev": 49.66443124430306, + "sum": 6428.866666666667 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.05, + "max": 1.0666666666666667, + "mean": 0.18645833333333334, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7458333333333333, + "non_zero_median": 0.9, + "percentile25": 0.0, + "percentile75": 0.05, + "percentile90": 0.9800000000000002, + "percentile99": 1.0246666666666664, + "percentile999": 1.062466666666667, + "range": 1.0666666666666667, + "samples": 64, + "stdev": 0.35756963037200723, + "sum": 11.933333333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2621.449999999999, + "max": 24445.2, + "mean": 7137.103125, + "median": 5247.733333333334, + "min": 2100.0666666666666, + "non_zero_mean": 7137.103125, + "non_zero_median": 5247.733333333334, + "percentile25": 4975.583333333334, + "percentile75": 7597.033333333333, + "percentile90": 11968.14, + "percentile99": 23421.407999999996, + "percentile999": 24342.82080000001, + "range": 22345.133333333335, + "samples": 64, + "stdev": 4373.439256749412, + "sum": 456774.6000000002 + }, + "pg_stat_database_tup_inserted": { + "iqr": 370.3666666666666, + "max": 836.2, + "mean": 534.2854166666666, + "median": 689.6666666666666, + "min": 0.0, + "non_zero_mean": 542.7661375661376, + "non_zero_median": 690.9333333333333, + "percentile25": 350.3, + "percentile75": 720.6666666666666, + "percentile90": 740.3533333333334, + "percentile99": 805.3299999999999, + "percentile999": 833.1130000000003, + "range": 836.2, + "samples": 64, + "stdev": 262.6926054038492, + "sum": 34194.26666666667 + }, + "pg_stat_database_tup_returned": { + "iqr": 2631.7666666666673, + "max": 30395.266666666666, + "mean": 9023.8125, + "median": 6595.633333333333, + "min": 3354.9333333333334, + "non_zero_mean": 9023.8125, + "non_zero_median": 6595.633333333333, + "percentile25": 6171.066666666667, + "percentile75": 8802.833333333334, + "percentile90": 16590.213333333337, + "percentile99": 28349.06866666666, + "percentile999": 30190.64686666668, + "range": 27040.333333333332, + "samples": 64, + "stdev": 5551.414284728517, + "sum": 577524.0 + }, + "pg_stat_database_tup_updated": { + "iqr": 8.15, + "max": 123.46666666666667, + "mean": 10.15, + "median": 1.1333333333333333, + "min": 0.0, + "non_zero_mean": 11.010169491525424, + "non_zero_median": 1.4, + "percentile25": 0.26666666666666666, + "percentile75": 8.416666666666668, + "percentile90": 15.166666666666668, + "percentile99": 122.71066666666667, + "percentile999": 123.39106666666667, + "range": 123.46666666666667, + "samples": 64, + "stdev": 25.846965169350316, + "sum": 649.5999999999999 + }, + "pg_stat_database_xact_commit": { + "iqr": 7.25, + "max": 144.66666666666666, + "mean": 58.98645833333333, + "median": 50.06666666666666, + "min": 12.2, + "non_zero_mean": 58.98645833333333, + "non_zero_median": 50.06666666666666, + "percentile25": 46.7, + "percentile75": 53.95, + "percentile90": 105.59333333333333, + "percentile99": 136.72866666666664, + "percentile999": 143.8728666666667, + "range": 132.46666666666667, + "samples": 64, + "stdev": 27.92160398177984, + "sum": 3775.1333333333337 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 4.266666666666671 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.004166666666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.09866666666666599, + "percentile999": 0.24986666666666793, + "range": 0.26666666666666666, + "samples": 64, + "stdev": 0.03333333333333333, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 21.51666666666668, + "max": 135.2, + "mean": 77.43333333333334, + "median": 82.4, + "min": 8.266666666666666, + "non_zero_mean": 77.43333333333334, + "non_zero_median": 82.4, + "percentile25": 67.64999999999999, + "percentile75": 89.16666666666667, + "percentile90": 107.09333333333333, + "percentile99": 134.402, + "percentile999": 135.12019999999998, + "range": 126.93333333333332, + "samples": 64, + "stdev": 26.78921839513468, + "sum": 4955.733333333334 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,8,11,9,10,14,18,21,20,19,23,22,24,25,26,27,29,33,28,32,31,30,35,34,36,37,38,39,43,40,41,42,44,45,46,47,48,49,52,50,53,51,55,54,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2021-12-21T21:25:48.733900Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "PASS", + "results": { + "created": "2021-12-21T21:25:49.557956Z 2021-12-21T21:25:49.451773Z 2021-12-21T21:25:49.417789Z 2021-12-21T21:25:48.771679Z 2021-12-21T21:25:48.759122Z 2021-12-21T21:25:48.753362Z 2021-12-21T21:25:48.733900Z 2021-12-21T21:25:50.727635Z 2021-12-21T21:25:49.605836Z 2021-12-21T21:25:49.565766Z 2021-12-21T21:30:02.773295Z 2021-12-21T21:30:02.642653Z 2021-12-21T21:30:02.606672Z 2021-12-21T21:30:02.192553Z 2021-12-21T21:30:02.187312Z 2021-12-21T21:30:02.180246Z 2021-12-21T21:30:02.078971Z 2021-12-21T21:30:02.061628Z 2021-12-21T21:30:02.045636Z 2021-12-21T21:30:01.986484Z 2021-12-21T21:32:56.667543Z 2021-12-21T21:32:56.653106Z 2021-12-21T21:32:56.141745Z 2021-12-21T21:32:56.128255Z 2021-12-21T21:32:56.073169Z 2021-12-21T21:32:56.070783Z 2021-12-21T21:32:56.069014Z 2021-12-21T21:32:56.063232Z 2021-12-21T21:32:56.057264Z 2021-12-21T21:32:56.053401Z 2021-12-21T21:36:02.372526Z 2021-12-21T21:36:02.264686Z 2021-12-21T21:36:01.284897Z 2021-12-21T21:36:01.266535Z 2021-12-21T21:36:01.262784Z 2021-12-21T21:36:01.260322Z 2021-12-21T21:36:01.256670Z 2021-12-21T21:36:01.237733Z 2021-12-21T21:36:01.232431Z 2021-12-21T21:36:01.134803Z 2021-12-21T21:38:59.672377Z 2021-12-21T21:38:59.518725Z 2021-12-21T21:38:59.235986Z 2021-12-21T21:38:59.219192Z 2021-12-21T21:38:59.203332Z 2021-12-21T21:38:59.081659Z 2021-12-21T21:38:59.078155Z 2021-12-21T21:38:59.040495Z 2021-12-21T21:38:59.039481Z 2021-12-21T21:38:58.945391Z", + "created_diff": 1.0718794, + "ended": "2021-12-21T21:41:40.852802Z", + "error_job_ids": " []", + "event_first": "2021-12-21T21:26:05.666657Z 2021-12-21T21:27:02.069162Z 2021-12-21T21:26:01.390583Z 2021-12-21T21:26:49.909092Z 2021-12-21T21:25:54.994412Z 2021-12-21T21:26:39.366149Z 2021-12-21T21:25:54.835606Z 2021-12-21T21:27:16.872163Z 2021-12-21T21:26:01.390583Z 2021-12-21T21:27:35.260119Z 2021-12-21T21:30:14.635269Z 2021-12-21T21:30:16.916495Z 2021-12-21T21:30:13.999167Z 2021-12-21T21:30:09.961457Z 2021-12-21T21:30:10.165401Z 2021-12-21T21:30:09.706652Z 2021-12-21T21:30:09.437996Z 2021-12-21T21:30:11.066433Z 2021-12-21T21:30:09.409335Z 2021-12-21T21:30:09.353905Z 2021-12-21T21:33:19.013830Z 2021-12-21T21:33:18.365355Z 2021-12-21T21:33:03.569350Z 2021-12-21T21:33:04.447500Z 2021-12-21T21:33:05.074456Z 2021-12-21T21:33:03.224867Z 2021-12-21T21:33:02.699371Z 2021-12-21T21:33:03.475904Z 2021-12-21T21:33:02.193431Z 2021-12-21T21:33:01.946860Z 2021-12-21T21:36:16.247127Z 2021-12-21T21:36:17.161401Z 2021-12-21T21:36:10.939507Z 2021-12-21T21:36:12.727228Z 2021-12-21T21:36:10.226538Z 2021-12-21T21:36:09.923248Z 2021-12-21T21:36:09.708457Z 2021-12-21T21:36:08.394345Z 2021-12-21T21:36:08.131469Z 2021-12-21T21:36:07.888751Z 2021-12-21T21:39:18.864096Z 2021-12-21T21:39:19.091438Z 2021-12-21T21:39:18.615350Z 2021-12-21T21:39:20.029332Z 2021-12-21T21:39:18.293079Z 2021-12-21T21:39:04.767506Z 2021-12-21T21:39:04.403077Z 2021-12-21T21:39:05.282666Z 2021-12-21T21:39:04.233025Z 2021-12-21T21:39:04.571579Z", + "event_last": "2021-12-21T21:28:40.085012Z 2021-12-21T21:29:23.532423Z 2021-12-21T21:28:36.374790Z 2021-12-21T21:29:16.245223Z 2021-12-21T21:28:28.922373Z 2021-12-21T21:29:08.942341Z 2021-12-21T21:28:27.627449Z 2021-12-21T21:29:33.252210Z 2021-12-21T21:28:35.769820Z 2021-12-21T21:29:43.480087Z 2021-12-21T21:32:40.065902Z 2021-12-21T21:32:38.662344Z 2021-12-21T21:32:36.798886Z 2021-12-21T21:32:34.441378Z 2021-12-21T21:32:35.154231Z 2021-12-21T21:32:35.057427Z 2021-12-21T21:32:33.902560Z 2021-12-21T21:32:35.942792Z 2021-12-21T21:32:33.033830Z 2021-12-21T21:32:34.928339Z 2021-12-21T21:35:35.315076Z 2021-12-21T21:35:35.904783Z 2021-12-21T21:35:25.976971Z 2021-12-21T21:35:28.136031Z 2021-12-21T21:35:28.072559Z 2021-12-21T21:35:26.411268Z 2021-12-21T21:35:25.428168Z 2021-12-21T21:35:27.553242Z 2021-12-21T21:35:26.570389Z 2021-12-21T21:35:24.430603Z 2021-12-21T21:38:40.516607Z 2021-12-21T21:38:42.647244Z 2021-12-21T21:38:36.607246Z 2021-12-21T21:38:38.817798Z 2021-12-21T21:38:37.260791Z 2021-12-21T21:38:36.581182Z 2021-12-21T21:38:35.642805Z 2021-12-21T21:38:35.514189Z 2021-12-21T21:38:33.335939Z 2021-12-21T21:38:35.361972Z 2021-12-21T21:41:40.753744Z 2021-12-21T21:41:40.591201Z 2021-12-21T21:41:40.852802Z 2021-12-21T21:41:40.591201Z 2021-12-21T21:41:38.026467Z 2021-12-21T21:41:29.348763Z 2021-12-21T21:41:27.632065Z 2021-12-21T21:41:29.650269Z 2021-12-21T21:41:27.300336Z 2021-12-21T21:41:28.229721Z", + "failed_job_ids": " []", + "finished": "2021-12-21T21:28:43.986547Z 2021-12-21T21:29:27.929193Z 2021-12-21T21:28:40.310671Z 2021-12-21T21:29:20.707132Z 2021-12-21T21:28:32.770785Z 2021-12-21T21:29:12.556968Z 2021-12-21T21:28:31.383328Z 2021-12-21T21:29:36.771115Z 2021-12-21T21:28:39.885629Z 2021-12-21T21:29:47.287578Z 2021-12-21T21:32:42.587933Z 2021-12-21T21:32:42.500369Z 2021-12-21T21:32:40.485065Z 2021-12-21T21:32:38.597108Z 2021-12-21T21:32:39.531414Z 2021-12-21T21:32:39.418996Z 2021-12-21T21:32:38.202012Z 2021-12-21T21:32:40.265768Z 2021-12-21T21:32:36.948202Z 2021-12-21T21:32:39.010574Z 2021-12-21T21:35:39.164761Z 2021-12-21T21:35:39.822558Z 2021-12-21T21:35:29.628217Z 2021-12-21T21:35:31.878395Z 2021-12-21T21:35:31.597583Z 2021-12-21T21:35:30.394068Z 2021-12-21T21:35:29.297114Z 2021-12-21T21:35:31.235552Z 2021-12-21T21:35:30.934863Z 2021-12-21T21:35:27.854366Z 2021-12-21T21:38:43.288638Z 2021-12-21T21:38:43.954149Z 2021-12-21T21:38:39.958378Z 2021-12-21T21:38:42.930389Z 2021-12-21T21:38:40.690117Z 2021-12-21T21:38:40.417865Z 2021-12-21T21:38:39.514320Z 2021-12-21T21:38:39.613678Z 2021-12-21T21:38:37.222864Z 2021-12-21T21:38:39.329007Z 2021-12-21T21:41:43.082454Z 2021-12-21T21:41:44.562575Z 2021-12-21T21:41:42.751630Z 2021-12-21T21:41:44.513134Z 2021-12-21T21:41:41.658284Z 2021-12-21T21:41:33.029802Z 2021-12-21T21:41:31.603684Z 2021-12-21T21:41:34.046523Z 2021-12-21T21:41:31.503277Z 2021-12-21T21:41:31.786340Z", + "finished_diff": 22.6605512, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 235.898806, + "mean": 161.99372219999998, + "min": 149.555716 + }, + "jobs_events_duration": { + "max": 154.984207, + "mean": 144.42728139999997, + "min": 128.219968 + }, + "jobs_events_lag": { + "max": -1.306905, + "mean": -3.74240246, + "min": -4.461909 + }, + "jobs_waiting": { + "max": 13.71594, + "mean": 3.0805780400000007, + "min": 0.853656 + }, + "modified": "2021-12-21T21:25:50.774550Z 2021-12-21T21:25:50.738904Z 2021-12-21T21:25:50.697688Z 2021-12-21T21:25:49.478474Z 2021-12-21T21:25:49.441578Z 2021-12-21T21:25:49.401286Z 2021-12-21T21:25:49.349884Z 2021-12-21T21:26:04.151315Z 2021-12-21T21:25:50.853347Z 2021-12-21T21:25:50.806734Z 2021-12-21T21:30:03.778508Z 2021-12-21T21:30:03.731445Z 2021-12-21T21:30:03.695269Z 2021-12-21T21:30:02.803945Z 2021-12-21T21:30:02.770001Z 2021-12-21T21:30:02.733947Z 2021-12-21T21:30:02.693220Z 2021-12-21T21:30:02.657565Z 2021-12-21T21:30:02.626059Z 2021-12-21T21:30:02.592308Z 2021-12-21T21:33:08.625014Z 2021-12-21T21:33:08.420085Z 2021-12-21T21:32:56.850856Z 2021-12-21T21:32:56.818987Z 2021-12-21T21:32:56.786533Z 2021-12-21T21:32:56.745367Z 2021-12-21T21:32:56.708902Z 2021-12-21T21:32:56.677392Z 2021-12-21T21:32:56.646086Z 2021-12-21T21:32:56.614389Z 2021-12-21T21:36:03.446027Z 2021-12-21T21:36:03.382380Z 2021-12-21T21:36:01.938381Z 2021-12-21T21:36:01.909593Z 2021-12-21T21:36:01.880462Z 2021-12-21T21:36:01.851319Z 2021-12-21T21:36:01.819141Z 2021-12-21T21:36:01.789304Z 2021-12-21T21:36:01.759610Z 2021-12-21T21:36:01.727537Z 2021-12-21T21:39:10.781517Z 2021-12-21T21:39:10.724920Z 2021-12-21T21:39:10.690077Z 2021-12-21T21:39:10.659914Z 2021-12-21T21:39:10.628632Z 2021-12-21T21:38:59.639788Z 2021-12-21T21:38:59.599766Z 2021-12-21T21:38:59.560984Z 2021-12-21T21:38:59.519257Z 2021-12-21T21:38:59.485092Z", + "modified_diff": 8.2026342, + "started": "2021-12-21T21:25:51.272148Z 2021-12-21T21:25:51.214301Z 2021-12-21T21:25:51.110189Z 2021-12-21T21:25:49.986575Z 2021-12-21T21:25:49.848656Z 2021-12-21T21:25:49.888473Z 2021-12-21T21:25:49.689423Z 2021-12-21T21:26:04.443575Z 2021-12-21T21:25:51.489392Z 2021-12-21T21:25:51.388772Z 2021-12-21T21:30:04.364074Z 2021-12-21T21:30:04.310002Z 2021-12-21T21:30:04.142994Z 2021-12-21T21:30:03.516792Z 2021-12-21T21:30:03.407240Z 2021-12-21T21:30:03.333692Z 2021-12-21T21:30:03.232107Z 2021-12-21T21:30:03.153366Z 2021-12-21T21:30:03.073962Z 2021-12-21T21:30:02.996373Z 2021-12-21T21:33:09.609045Z 2021-12-21T21:33:09.305890Z 2021-12-21T21:32:57.662511Z 2021-12-21T21:32:57.634506Z 2021-12-21T21:32:57.442938Z 2021-12-21T21:32:57.375486Z 2021-12-21T21:32:57.285621Z 2021-12-21T21:32:57.190796Z 2021-12-21T21:32:57.117381Z 2021-12-21T21:32:57.030234Z 2021-12-21T21:36:04.095760Z 2021-12-21T21:36:03.898346Z 2021-12-21T21:36:02.814651Z 2021-12-21T21:36:02.722286Z 2021-12-21T21:36:02.645066Z 2021-12-21T21:36:02.535950Z 2021-12-21T21:36:02.437993Z 2021-12-21T21:36:02.347701Z 2021-12-21T21:36:02.263465Z 2021-12-21T21:36:02.193278Z 2021-12-21T21:39:11.815558Z 2021-12-21T21:39:11.601369Z 2021-12-21T21:39:11.393265Z 2021-12-21T21:39:11.356093Z 2021-12-21T21:39:11.146249Z 2021-12-21T21:39:00.154385Z 2021-12-21T21:39:00.079107Z 2021-12-21T21:38:59.986093Z 2021-12-21T21:38:59.893137Z 2021-12-21T21:38:59.818596Z", + "started_diff": 8.520021599999998, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2021-12-21T21:44:24.204119+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-launching.json new file mode 100644 index 00000000..fde81c4a --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-21T20_47_48_348_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2021-12-21T22:29:08.877471+00:00", + "golden": "true", + "id": "run-2021-12-21T20_47_48_348_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 18614.899999999998, + "max": 29739.533333333333, + "mean": 10081.159459459459, + "median": 5245.733333333334, + "min": 1536.0, + "non_zero_mean": 10081.159459459459, + "non_zero_median": 5245.733333333334, + "percentile25": 1739.2166666666667, + "percentile75": 20354.116666666665, + "percentile90": 23368.833333333336, + "percentile99": 27816.61599999999, + "percentile999": 29547.241600000016, + "range": 28203.533333333333, + "samples": 74, + "stdev": 9329.324446596936, + "sum": 746005.7999999999 + } + }, + "cpu": { + "system": { + "iqr": 1.4208333333333358, + "max": 1.7680000000000027, + "mean": 0.6994414414414414, + "median": 0.34300000000000164, + "min": 0.026666666666665156, + "non_zero_mean": 0.6994414414414414, + "non_zero_median": 0.34300000000000164, + "percentile25": 0.039499999999998175, + "percentile75": 1.460333333333334, + "percentile90": 1.6557333333333335, + "percentile99": 1.7641066666666645, + "percentile999": 1.7676106666666689, + "range": 1.7413333333333376, + "samples": 74, + "stdev": 0.7007017836958929, + "sum": 51.75866666666665 + }, + "user": { + "iqr": 5.751999999999954, + "max": 6.52533333333334, + "mean": 2.958162162162162, + "median": 1.2866666666666786, + "min": 0.07533333333331028, + "non_zero_mean": 2.958162162162162, + "non_zero_median": 1.2866666666666786, + "percentile25": 0.32133333333334807, + "percentile75": 6.073333333333301, + "percentile90": 6.317533333333337, + "percentile99": 6.500999999999985, + "percentile999": 6.522900000000005, + "range": 6.45000000000003, + "samples": 74, + "stdev": 2.753861286542908, + "sum": 218.904 + } + }, + "entropy_available_bits": { + "iqr": 4.583333333333333, + "max": 214.86666666666667, + "mean": 24.614414414414416, + "median": 1.3333333333333335, + "min": 0.0, + "non_zero_mean": 33.73086419753086, + "non_zero_median": 3.666666666666667, + "percentile25": 0.0, + "percentile75": 4.583333333333333, + "percentile90": 149.14666666666724, + "percentile99": 214.672, + "percentile999": 214.84720000000002, + "range": 214.86666666666667, + "samples": 74, + "stdev": 65.62899484539186, + "sum": 1821.4666666666658 + }, + "memory": { + "used": { + "iqr": 5177531392.0, + "max": 19091206144.0, + "mean": 8905695730.162163, + "median": 6720962560.0, + "min": 5248352256.0, + "non_zero_mean": 8905695730.162163, + "non_zero_median": 6720962560.0, + "percentile25": 6212104192.0, + "percentile75": 11389635584.0, + "percentile90": 15411801292.800001, + "percentile99": 18624738713.6, + "percentile999": 19044559400.960003, + "range": 13842853888.0, + "samples": 74, + "stdev": 3857785263.513216, + "sum": 659021484032.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.005405405405405406, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2, + "non_zero_median": 0.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.1693333333333328, + "percentile999": 0.25693333333333424, + "range": 0.26666666666666666, + "samples": 74, + "stdev": 0.034467917726792176, + "sum": 0.4 + } + }, + "writes_completed": { + "total": { + "iqr": 822.7666666666667, + "max": 1666.4, + "mean": 468.25405405405405, + "median": 104.20000000000002, + "min": 0.33333333333333337, + "non_zero_mean": 468.25405405405405, + "non_zero_median": 104.20000000000002, + "percentile25": 3.9, + "percentile75": 826.6666666666666, + "percentile90": 1527.3266666666666, + "percentile99": 1651.8486666666665, + "percentile999": 1664.944866666667, + "range": 1666.0666666666668, + "samples": 74, + "stdev": 589.3236574006479, + "sum": 34650.8 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2104.55, + "max": 3688.133333333333, + "mean": 1735.200900900901, + "median": 1303.3666666666668, + "min": 308.1333333333333, + "non_zero_mean": 1735.200900900901, + "non_zero_median": 1303.3666666666668, + "percentile25": 648.25, + "percentile75": 2752.8, + "percentile90": 3156.8866666666668, + "percentile99": 3638.785333333333, + "percentile999": 3683.1985333333337, + "range": 3380.0, + "samples": 74, + "stdev": 1139.1103878897181, + "sum": 128404.86666666668 + } + }, + "cpu": { + "system": { + "iqr": 0.204833333333334, + "max": 0.40266666666666706, + "mean": 0.13572972972972974, + "median": 0.09066666666666687, + "min": 0.010666666666666439, + "non_zero_mean": 0.13572972972972974, + "non_zero_median": 0.09066666666666687, + "percentile25": 0.024666666666666136, + "percentile75": 0.22950000000000015, + "percentile90": 0.29640000000000055, + "percentile99": 0.3832, + "percentile999": 0.4007200000000005, + "range": 0.3920000000000006, + "samples": 74, + "stdev": 0.12098122016956153, + "sum": 10.044000000000002 + }, + "user": { + "iqr": 0.30916666666666687, + "max": 0.8173333333333327, + "mean": 0.20979279279279284, + "median": 0.15100000000000052, + "min": 0.013333333333333523, + "non_zero_mean": 0.20979279279279284, + "non_zero_median": 0.15100000000000052, + "percentile25": 0.040166666666666136, + "percentile75": 0.349333333333333, + "percentile90": 0.43400000000000083, + "percentile99": 0.7482266666666664, + "percentile999": 0.8104226666666667, + "range": 0.8039999999999992, + "samples": 74, + "stdev": 0.1960259643497442, + "sum": 15.52466666666667 + } + }, + "entropy_available_bits": { + "iqr": 3.2333333333333334, + "max": 212.93333333333334, + "mean": 12.541441441441442, + "median": 0.3333333333333333, + "min": 0.0, + "non_zero_mean": 23.796581196581197, + "non_zero_median": 3.1333333333333333, + "percentile25": 0.0, + "percentile75": 3.2333333333333334, + "percentile90": 4.3133333333333335, + "percentile99": 212.836, + "percentile999": 212.9236, + "range": 212.93333333333334, + "samples": 74, + "stdev": 47.69707552909011, + "sum": 928.0666666666668 + }, + "memory": { + "used": { + "iqr": 391916544.0, + "max": 1157726208.0, + "mean": 573436125.4054054, + "median": 393072640.0, + "min": 362995712.0, + "non_zero_mean": 573436125.4054054, + "non_zero_median": 393072640.0, + "percentile25": 376092672.0, + "percentile75": 768009216.0, + "percentile90": 1008605593.6, + "percentile99": 1101007380.4799998, + "percentile999": 1152054325.2480006, + "range": 794730496.0, + "samples": 74, + "stdev": 256677632.35722935, + "sum": 42434273280.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 7645.866666666667, + "max": 56130004.2, + "mean": 776528.5900900902, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2873155.7833333337, + "non_zero_median": 56797.86666666667, + "percentile25": 0.0, + "percentile75": 7645.866666666667, + "percentile90": 62095.36, + "percentile99": 15291050.104666444, + "percentile999": 52046108.790467046, + "range": 56130004.2, + "samples": 74, + "stdev": 6522999.813002758, + "sum": 57463115.66666667 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.025689, + "mean": 0.0017087027027027028, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.010537, + "non_zero_median": 0.008697, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0034677000000000054, + "percentile99": 0.025092589999999994, + "percentile999": 0.025629359000000004, + "range": 0.025689, + "samples": 74, + "stdev": 0.005375599095582831, + "sum": 0.126444 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.623977, + "max": 8.893244, + "mean": 0.609761472972973, + "median": 0.0, + "min": -0.001046, + "non_zero_mean": 1.367343909090909, + "non_zero_median": 0.859521, + "percentile25": 0.0, + "percentile75": 0.623977, + "percentile90": 1.4322183, + "percentile99": 7.724363619999993, + "percentile999": 8.776355962000011, + "range": 8.89429, + "samples": 74, + "stdev": 1.4832847981842803, + "sum": 45.12234900000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.4, + "mean": 0.13963963963963963, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.583333333333333, + "non_zero_median": 2.8666666666666663, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.0593333333333312, + "percentile999": 3.3659333333333366, + "range": 3.4, + "samples": 74, + "stdev": 0.6193401426916679, + "sum": 10.333333333333332 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9990.466666666667, + "mean": 531.1495495495495, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9826.266666666666, + "non_zero_median": 9990.066666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9990.418, + "percentile999": 9990.461800000001, + "range": 9990.466666666667, + "samples": 74, + "stdev": 2238.112242539519, + "sum": 39305.066666666666 + }, + "pg_stat_database_blks_hit": { + "iqr": 28561.683333333327, + "max": 103598.6, + "mean": 22555.09279279279, + "median": 14909.966666666667, + "min": 1368.4666666666667, + "non_zero_mean": 22555.09279279279, + "non_zero_median": 14909.966666666667, + "percentile25": 7682.616666666667, + "percentile75": 36244.299999999996, + "percentile90": 48829.09333333335, + "percentile99": 93139.01399999994, + "percentile999": 102552.6414000001, + "range": 102230.13333333335, + "samples": 74, + "stdev": 21726.597916401508, + "sum": 1669076.866666667 + }, + "pg_stat_database_blks_read": { + "iqr": 3.3333333333333335, + "max": 13.6, + "mean": 2.1837837837837837, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.672727272727273, + "non_zero_median": 3.0666666666666664, + "percentile25": 0.0, + "percentile75": 3.3333333333333335, + "percentile90": 6.720000000000002, + "percentile99": 12.918666666666663, + "percentile999": 13.531866666666673, + "range": 13.6, + "samples": 74, + "stdev": 3.195731757922101, + "sum": 161.59999999999997 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 74, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.3333333333333335, + "max": 13.6, + "mean": 2.1837837837837837, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.672727272727273, + "non_zero_median": 3.0666666666666664, + "percentile25": 0.0, + "percentile75": 3.3333333333333335, + "percentile90": 6.720000000000002, + "percentile99": 12.918666666666663, + "percentile999": 13.531866666666673, + "range": 13.6, + "samples": 74, + "stdev": 3.195731757922101, + "sum": 161.59999999999997 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.7333333333333334, + "mean": 0.2072072072072072, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.0222222222222221, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9800000000000002, + "percentile99": 1.3926666666666647, + "percentile999": 1.6992666666666698, + "range": 1.7333333333333334, + "samples": 74, + "stdev": 0.43325500768312464, + "sum": 15.333333333333332 + }, + "pg_stat_database_tup_fetched": { + "iqr": 16113.416666666666, + "max": 52148.933333333334, + "mean": 11981.537837837837, + "median": 8148.433333333333, + "min": 677.6666666666666, + "non_zero_mean": 11981.537837837837, + "non_zero_median": 8148.433333333333, + "percentile25": 3930.3999999999996, + "percentile75": 20043.816666666666, + "percentile90": 27120.57333333335, + "percentile99": 47183.278666666636, + "percentile999": 51652.36786666671, + "range": 51471.26666666667, + "samples": 74, + "stdev": 11260.642301684833, + "sum": 886633.7999999999 + }, + "pg_stat_database_tup_inserted": { + "iqr": 28.083333333333332, + "max": 106.06666666666666, + "mean": 17.08828828828829, + "median": 5.699999999999999, + "min": 0.0, + "non_zero_mean": 26.904964539007093, + "non_zero_median": 21.866666666666667, + "percentile25": 0.0, + "percentile75": 28.083333333333332, + "percentile90": 45.553333333333335, + "percentile99": 97.40399999999995, + "percentile999": 105.20040000000007, + "range": 106.06666666666666, + "samples": 74, + "stdev": 23.76625931140615, + "sum": 1264.5333333333335 + }, + "pg_stat_database_tup_returned": { + "iqr": 22982.083333333332, + "max": 77672.86666666667, + "mean": 19044.508108108108, + "median": 12773.8, + "min": 1380.8, + "non_zero_mean": 19044.508108108108, + "non_zero_median": 12773.8, + "percentile25": 6693.183333333333, + "percentile75": 29675.266666666666, + "percentile90": 42421.94666666668, + "percentile99": 75544.18666666666, + "percentile999": 77459.99866666668, + "range": 76292.06666666667, + "samples": 74, + "stdev": 17359.051072865255, + "sum": 1409293.6000000003 + }, + "pg_stat_database_tup_updated": { + "iqr": 30.683333333333334, + "max": 78.0, + "mean": 17.601801801801802, + "median": 11.7, + "min": 0.0, + "non_zero_mean": 20.352083333333333, + "non_zero_median": 14.166666666666668, + "percentile25": 0.26666666666666666, + "percentile75": 30.95, + "percentile90": 47.34, + "percentile99": 66.3686666666666, + "percentile999": 76.83686666666678, + "range": 78.0, + "samples": 74, + "stdev": 19.89906638463137, + "sum": 1302.5333333333335 + }, + "pg_stat_database_xact_commit": { + "iqr": 283.7833333333333, + "max": 579.4, + "mean": 158.52972972972972, + "median": 129.06666666666666, + "min": 3.4, + "non_zero_mean": 158.52972972972972, + "non_zero_median": 129.06666666666666, + "percentile25": 9.383333333333333, + "percentile75": 293.16666666666663, + "percentile90": 382.33333333333337, + "percentile99": 554.7746666666665, + "percentile999": 576.9374666666669, + "range": 576.0, + "samples": 74, + "stdev": 161.20773702568587, + "sum": 11731.2 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 74, + "stdev": 0.0, + "sum": 4.933333333333335 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.007207207207207207, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.26666666666666666, + "percentile999": 0.26666666666666666, + "range": 0.26666666666666666, + "samples": 74, + "stdev": 0.04353842239173748, + "sum": 0.5333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 100.49999999999999, + "max": 164.4, + "mean": 60.25315315315315, + "median": 39.06666666666666, + "min": 1.4666666666666666, + "non_zero_mean": 60.25315315315315, + "non_zero_median": 39.06666666666666, + "percentile25": 5.133333333333334, + "percentile75": 105.63333333333333, + "percentile90": 135.75333333333333, + "percentile99": 157.39199999999997, + "percentile999": 163.69920000000008, + "range": 162.93333333333334, + "samples": 74, + "stdev": 54.05761060104332, + "sum": 4458.733333333334 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,64,60,74,62,67,66,69,65,72,70,63,75,78,68,76,82,80,77,89,81,79,85,86,95,88,83,96,92,90,93,94,97,100,104,105,106,98,99,102,101,107,108,103,109,112,111,113,110,114,117,115,119,122,120,121,125,123,116,128,129,124,131,135,137,138,132,139,141,130,126,134,143,145,147,148,150,152,144,118,149,154,153,142,157,136,158,159,161,162,163,156,166,165,168,167,160,164,170,169,172,175,171,173,174,177,178,176,179,180,182,184,187,185,183,192,190,188,193,181,186,195,196,198,197,194,199,191,189,200,204,207,206,201,202,203,205,208,209,211,212,215,210,214,219,216,213,223,222,218,220,217,224,229,225,228,226,233,227,232,234,230,221,236,237,238,241,244,239,248,243,235,240,246,249,231,253,247,254,256,250,257,259,252,260,261,262,264,245,263,258,265,255,242,267,269,251,268,266,270,271,272,273,280,275,279,278,274,289,285,276,277,290,281,287,291,288,283,296,284,286,282,297,298,305,292,294,293,299,300,304,301,303,295,307,302,308,313,306,315,310,318,309,311,314,316,312,322,320,321,324,325,319,326,327,330,323,329,331,328,334,335,339,333,337,336,346,340,342,348,317,341,344,349,345,353,351,343,352,332,354,355,356,357,359,360,358,361,363,347,338,364,362,365,366,350,369,368,367,370,371,372,374,381,383,382,373,377,376,380,398,389,379,384,375,378,386,391,388,395,385,390,394,404,396,392,387,393,400,402,403,397,399,401,406,412,410,413,405,407,409,411,415,418,416,420,414,417,408,419,423,421,424,426,425,429,428,422,427,430,432,431,433,438,436,435,437,439,445,434,442,447,441,448,443,450,440,444,446,451,449,452,457,458,454,455,461,456,453,463,462,459,460,464,465,466,467,468,470,469,475,476,474,480,488,487,471,481,473,483,479,477,482,485,484,489,495,486,478,493,472,491,494,492,496,490,497,499,498,500,511,502,504,509,508,507,505,503,501,510,506,513,512,516,520,519,514,518,521,522,515,527,517,528,532,531,537,523,524,534,530,536,526,525,533,529,538,535,551,546,548,541,554,540,549,542,544,550,543,547,553,545,552,558,557,555,560,559,565,539,564,563,568,562,561,556,566,569,567,570", + "job_template_id": 11, + "project_id": 10, + "started": "2021-12-21T21:46:18.621267Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2021-12-21T21:46:23.273449Z 2021-12-21T21:46:23.191394Z 2021-12-21T21:46:23.032512Z 2021-12-21T21:46:22.971894Z 2021-12-21T21:46:22.965670Z 2021-12-21T21:46:22.840250Z 2021-12-21T21:46:22.482779Z 2021-12-21T21:46:21.976244Z 2021-12-21T21:46:21.951844Z 2021-12-21T21:46:21.850761Z 2021-12-21T21:46:21.789285Z 2021-12-21T21:46:21.530839Z 2021-12-21T21:46:21.520651Z 2021-12-21T21:46:21.030663Z 2021-12-21T21:46:21.010494Z 2021-12-21T21:46:20.884222Z 2021-12-21T21:46:20.701850Z 2021-12-21T21:46:20.628658Z 2021-12-21T21:46:20.584011Z 2021-12-21T21:46:20.519814Z 2021-12-21T21:46:20.363598Z 2021-12-21T21:46:20.050635Z 2021-12-21T21:46:20.020556Z 2021-12-21T21:46:19.973920Z 2021-12-21T21:46:19.855444Z 2021-12-21T21:46:19.517541Z 2021-12-21T21:46:19.489906Z 2021-12-21T21:46:19.467087Z 2021-12-21T21:46:19.338268Z 2021-12-21T21:46:19.176471Z 2021-12-21T21:46:19.171164Z 2021-12-21T21:46:19.032003Z 2021-12-21T21:46:18.942739Z 2021-12-21T21:46:18.851175Z 2021-12-21T21:46:18.621267Z 2021-12-21T21:46:40.642452Z 2021-12-21T21:46:39.877716Z 2021-12-21T21:46:37.844077Z 2021-12-21T21:46:37.267913Z 2021-12-21T21:46:36.955119Z 2021-12-21T21:46:36.735582Z 2021-12-21T21:46:36.453656Z 2021-12-21T21:46:36.070530Z 2021-12-21T21:46:36.063037Z 2021-12-21T21:46:35.514242Z 2021-12-21T21:46:35.249579Z 2021-12-21T21:46:34.679365Z 2021-12-21T21:46:33.631016Z 2021-12-21T21:46:32.018015Z 2021-12-21T21:46:31.592457Z 2021-12-21T21:46:30.282923Z 2021-12-21T21:46:30.192085Z 2021-12-21T21:46:30.116576Z 2021-12-21T21:46:30.037558Z 2021-12-21T21:46:30.015716Z 2021-12-21T21:46:29.672679Z 2021-12-21T21:46:29.555459Z 2021-12-21T21:46:29.440255Z 2021-12-21T21:46:29.380022Z 2021-12-21T21:46:29.078138Z 2021-12-21T21:46:28.784226Z 2021-12-21T21:46:28.587232Z 2021-12-21T21:46:28.528860Z 2021-12-21T21:46:28.299161Z 2021-12-21T21:46:28.266002Z 2021-12-21T21:46:28.137171Z 2021-12-21T21:46:28.010798Z 2021-12-21T21:46:27.765925Z 2021-12-21T21:46:27.635958Z 2021-12-21T21:46:27.484577Z 2021-12-21T21:46:27.253029Z 2021-12-21T21:46:27.046154Z 2021-12-21T21:46:26.888590Z 2021-12-21T21:46:26.745473Z 2021-12-21T21:46:26.643564Z 2021-12-21T21:46:26.561014Z 2021-12-21T21:46:26.572541Z 2021-12-21T21:46:26.510036Z 2021-12-21T21:46:26.414873Z 2021-12-21T21:46:26.328392Z 2021-12-21T21:46:26.033696Z 2021-12-21T21:46:25.904524Z 2021-12-21T21:46:25.751950Z 2021-12-21T21:46:25.670229Z 2021-12-21T21:46:25.629025Z 2021-12-21T21:46:25.577881Z 2021-12-21T21:46:25.382110Z 2021-12-21T21:46:25.264791Z 2021-12-21T21:46:25.160541Z 2021-12-21T21:46:25.155187Z 2021-12-21T21:46:25.048668Z 2021-12-21T21:46:24.467458Z 2021-12-21T21:46:24.180860Z 2021-12-21T21:46:23.944279Z 2021-12-21T21:46:23.937492Z 2021-12-21T21:46:23.730620Z 2021-12-21T21:46:23.692463Z 2021-12-21T21:46:23.498414Z 2021-12-21T21:46:23.438107Z 2021-12-21T21:46:23.342146Z 2021-12-21T21:51:09.024062Z 2021-12-21T21:51:08.229235Z 2021-12-21T21:51:08.166625Z 2021-12-21T21:51:07.540199Z 2021-12-21T21:51:07.379386Z 2021-12-21T21:51:07.053329Z 2021-12-21T21:51:06.300572Z 2021-12-21T21:51:06.223859Z 2021-12-21T21:51:06.158138Z 2021-12-21T21:51:05.956170Z 2021-12-21T21:51:05.927549Z 2021-12-21T21:51:05.473469Z 2021-12-21T21:51:05.089786Z 2021-12-21T21:51:04.971218Z 2021-12-21T21:51:04.865880Z 2021-12-21T21:51:04.777293Z 2021-12-21T21:51:04.567665Z 2021-12-21T21:51:04.414404Z 2021-12-21T21:51:04.203809Z 2021-12-21T21:51:04.153704Z 2021-12-21T21:51:04.038098Z 2021-12-21T21:51:03.741549Z 2021-12-21T21:51:03.594647Z 2021-12-21T21:51:03.431872Z 2021-12-21T21:51:03.406379Z 2021-12-21T21:51:03.293656Z 2021-12-21T21:51:03.088238Z 2021-12-21T21:51:02.989641Z 2021-12-21T21:51:02.795298Z 2021-12-21T21:51:02.721350Z 2021-12-21T21:51:02.381529Z 2021-12-21T21:51:02.370920Z 2021-12-21T21:51:02.370444Z 2021-12-21T21:51:02.300358Z 2021-12-21T21:51:02.297055Z 2021-12-21T21:51:01.534559Z 2021-12-21T21:51:01.328559Z 2021-12-21T21:51:01.045802Z 2021-12-21T21:51:00.998060Z 2021-12-21T21:51:00.845897Z 2021-12-21T21:51:00.870664Z 2021-12-21T21:51:00.378854Z 2021-12-21T21:51:00.094861Z 2021-12-21T21:50:59.893597Z 2021-12-21T21:50:59.883706Z 2021-12-21T21:50:59.604673Z 2021-12-21T21:50:59.572088Z 2021-12-21T21:50:59.397833Z 2021-12-21T21:50:59.377078Z 2021-12-21T21:50:59.328066Z 2021-12-21T21:50:59.309471Z 2021-12-21T21:50:59.192763Z 2021-12-21T21:50:58.830774Z 2021-12-21T21:50:58.768860Z 2021-12-21T21:50:58.449085Z 2021-12-21T21:50:58.002632Z 2021-12-21T21:50:57.898651Z 2021-12-21T21:50:57.892137Z 2021-12-21T21:50:57.797220Z 2021-12-21T21:50:57.793347Z 2021-12-21T21:50:57.719028Z 2021-12-21T21:50:57.558644Z 2021-12-21T21:50:57.022081Z 2021-12-21T21:50:56.714762Z 2021-12-21T21:50:56.642563Z 2021-12-21T21:50:56.594104Z 2021-12-21T21:50:56.464845Z 2021-12-21T21:50:56.426149Z 2021-12-21T21:50:56.316453Z 2021-12-21T21:50:56.305121Z 2021-12-21T21:50:56.261298Z 2021-12-21T21:50:55.591108Z 2021-12-21T21:50:55.540380Z 2021-12-21T21:50:55.536414Z 2021-12-21T21:50:55.516132Z 2021-12-21T21:50:55.009278Z 2021-12-21T21:50:54.903117Z 2021-12-21T21:50:54.736494Z 2021-12-21T21:50:54.608978Z 2021-12-21T21:50:54.555263Z 2021-12-21T21:50:54.544663Z 2021-12-21T21:50:54.525464Z 2021-12-21T21:50:54.147709Z 2021-12-21T21:50:54.132186Z 2021-12-21T21:50:54.069687Z 2021-12-21T21:50:54.029835Z 2021-12-21T21:50:53.911812Z 2021-12-21T21:50:53.853501Z 2021-12-21T21:50:53.574715Z 2021-12-21T21:50:53.256011Z 2021-12-21T21:50:53.084773Z 2021-12-21T21:50:52.887315Z 2021-12-21T21:50:52.262175Z 2021-12-21T21:50:52.130845Z 2021-12-21T21:50:52.056244Z 2021-12-21T21:50:51.949472Z 2021-12-21T21:50:51.933384Z 2021-12-21T21:50:51.717734Z 2021-12-21T21:50:51.599785Z 2021-12-21T21:50:51.523375Z 2021-12-21T21:55:00.825171Z 2021-12-21T21:54:59.180166Z 2021-12-21T21:54:58.717594Z 2021-12-21T21:54:58.705289Z 2021-12-21T21:54:57.438907Z 2021-12-21T21:54:56.753015Z 2021-12-21T21:54:56.661835Z 2021-12-21T21:54:56.447947Z 2021-12-21T21:54:56.398476Z 2021-12-21T21:54:55.765676Z 2021-12-21T21:54:55.689815Z 2021-12-21T21:54:55.502996Z 2021-12-21T21:54:55.381373Z 2021-12-21T21:54:55.361085Z 2021-12-21T21:54:55.179925Z 2021-12-21T21:54:55.171240Z 2021-12-21T21:54:54.755104Z 2021-12-21T21:54:54.544450Z 2021-12-21T21:54:54.484091Z 2021-12-21T21:54:54.114950Z 2021-12-21T21:54:54.060228Z 2021-12-21T21:54:53.931578Z 2021-12-21T21:54:53.646900Z 2021-12-21T21:54:53.596516Z 2021-12-21T21:54:53.557792Z 2021-12-21T21:54:53.369709Z 2021-12-21T21:54:53.189744Z 2021-12-21T21:54:53.123218Z 2021-12-21T21:54:52.939284Z 2021-12-21T21:54:52.912788Z 2021-12-21T21:54:52.728552Z 2021-12-21T21:54:52.129824Z 2021-12-21T21:54:51.946988Z 2021-12-21T21:54:51.804720Z 2021-12-21T21:54:51.638271Z 2021-12-21T21:54:51.566301Z 2021-12-21T21:54:51.536827Z 2021-12-21T21:54:51.342696Z 2021-12-21T21:54:51.246397Z 2021-12-21T21:54:51.096877Z 2021-12-21T21:54:51.090002Z 2021-12-21T21:54:51.013177Z 2021-12-21T21:54:51.015058Z 2021-12-21T21:54:50.974930Z 2021-12-21T21:54:50.865463Z 2021-12-21T21:54:50.256370Z 2021-12-21T21:54:50.164897Z 2021-12-21T21:54:49.948870Z 2021-12-21T21:54:49.936405Z 2021-12-21T21:54:49.902186Z 2021-12-21T21:54:49.879770Z 2021-12-21T21:54:49.868441Z 2021-12-21T21:54:49.823159Z 2021-12-21T21:54:49.699247Z 2021-12-21T21:54:49.655363Z 2021-12-21T21:54:49.535448Z 2021-12-21T21:54:49.497419Z 2021-12-21T21:54:49.281710Z 2021-12-21T21:54:49.258420Z 2021-12-21T21:54:49.247894Z 2021-12-21T21:54:49.129576Z 2021-12-21T21:54:48.955977Z 2021-12-21T21:54:48.508003Z 2021-12-21T21:54:48.507635Z 2021-12-21T21:54:48.347164Z 2021-12-21T21:54:48.126811Z 2021-12-21T21:54:48.104930Z 2021-12-21T21:54:48.053210Z 2021-12-21T21:54:48.049642Z 2021-12-21T21:54:47.992992Z 2021-12-21T21:54:47.899132Z 2021-12-21T21:54:47.746937Z 2021-12-21T21:54:47.610745Z 2021-12-21T21:54:47.554368Z 2021-12-21T21:54:47.541060Z 2021-12-21T21:54:47.465055Z 2021-12-21T21:54:47.408470Z 2021-12-21T21:54:47.215615Z 2021-12-21T21:54:47.049169Z 2021-12-21T21:54:47.000793Z 2021-12-21T21:54:46.595186Z 2021-12-21T21:54:46.473830Z 2021-12-21T21:54:46.426208Z 2021-12-21T21:54:46.425020Z 2021-12-21T21:54:46.411741Z 2021-12-21T21:54:46.394707Z 2021-12-21T21:54:46.331042Z 2021-12-21T21:54:46.290952Z 2021-12-21T21:54:46.006370Z 2021-12-21T21:54:45.911747Z 2021-12-21T21:54:45.908661Z 2021-12-21T21:54:45.876808Z 2021-12-21T21:54:45.849682Z 2021-12-21T21:54:45.764281Z 2021-12-21T21:54:45.641098Z 2021-12-21T21:54:45.439708Z 2021-12-21T21:54:45.402047Z 2021-12-21T21:54:45.370467Z 2021-12-21T21:54:44.903734Z 2021-12-21T21:54:44.228287Z 2021-12-21T21:59:00.749248Z 2021-12-21T21:59:00.206690Z 2021-12-21T21:59:00.013375Z 2021-12-21T21:58:59.460279Z 2021-12-21T21:58:59.423533Z 2021-12-21T21:58:59.019264Z 2021-12-21T21:58:58.903893Z 2021-12-21T21:58:58.414994Z 2021-12-21T21:58:58.278165Z 2021-12-21T21:58:57.846788Z 2021-12-21T21:58:57.730901Z 2021-12-21T21:58:57.706685Z 2021-12-21T21:58:57.565463Z 2021-12-21T21:58:57.484628Z 2021-12-21T21:58:57.479918Z 2021-12-21T21:58:57.397329Z 2021-12-21T21:58:57.401197Z 2021-12-21T21:58:57.313410Z 2021-12-21T21:58:57.142823Z 2021-12-21T21:58:57.015155Z 2021-12-21T21:58:56.730954Z 2021-12-21T21:58:56.503418Z 2021-12-21T21:58:56.459674Z 2021-12-21T21:58:56.426165Z 2021-12-21T21:58:56.331814Z 2021-12-21T21:58:56.254793Z 2021-12-21T21:58:56.236676Z 2021-12-21T21:58:56.064285Z 2021-12-21T21:58:56.013905Z 2021-12-21T21:58:56.013436Z 2021-12-21T21:58:55.786538Z 2021-12-21T21:58:55.632326Z 2021-12-21T21:58:55.565264Z 2021-12-21T21:58:55.515692Z 2021-12-21T21:58:55.486946Z 2021-12-21T21:58:55.438956Z 2021-12-21T21:58:55.430828Z 2021-12-21T21:58:55.299658Z 2021-12-21T21:58:55.210570Z 2021-12-21T21:58:55.131946Z 2021-12-21T21:58:55.067996Z 2021-12-21T21:58:54.570281Z 2021-12-21T21:58:54.442572Z 2021-12-21T21:58:54.411531Z 2021-12-21T21:58:54.350294Z 2021-12-21T21:58:54.329980Z 2021-12-21T21:58:54.251394Z 2021-12-21T21:58:54.244160Z 2021-12-21T21:58:54.221523Z 2021-12-21T21:58:53.779041Z 2021-12-21T21:58:53.668214Z 2021-12-21T21:58:53.650542Z 2021-12-21T21:58:53.593150Z 2021-12-21T21:58:53.582645Z 2021-12-21T21:58:53.513411Z 2021-12-21T21:58:53.426127Z 2021-12-21T21:58:53.389419Z 2021-12-21T21:58:53.344298Z 2021-12-21T21:58:53.292559Z 2021-12-21T21:58:52.980533Z 2021-12-21T21:58:52.982056Z 2021-12-21T21:58:52.926433Z 2021-12-21T21:58:52.924278Z 2021-12-21T21:58:52.907794Z 2021-12-21T21:58:52.840180Z 2021-12-21T21:58:52.771878Z 2021-12-21T21:58:52.723176Z 2021-12-21T21:58:52.404134Z 2021-12-21T21:58:52.289040Z 2021-12-21T21:58:52.250785Z 2021-12-21T21:58:52.220952Z 2021-12-21T21:58:52.078891Z 2021-12-21T21:58:52.061145Z 2021-12-21T21:58:51.993709Z 2021-12-21T21:58:51.949279Z 2021-12-21T21:58:51.827940Z 2021-12-21T21:58:51.812445Z 2021-12-21T21:58:51.714637Z 2021-12-21T21:58:51.692946Z 2021-12-21T21:58:51.635864Z 2021-12-21T21:58:51.628850Z 2021-12-21T21:58:51.572767Z 2021-12-21T21:58:51.511124Z 2021-12-21T21:58:51.276086Z 2021-12-21T21:58:51.182440Z 2021-12-21T21:58:51.177626Z 2021-12-21T21:58:51.086175Z 2021-12-21T21:58:51.056601Z 2021-12-21T21:58:51.005015Z 2021-12-21T21:58:50.920920Z 2021-12-21T21:58:50.716878Z 2021-12-21T21:58:50.627885Z 2021-12-21T21:58:50.599296Z 2021-12-21T21:58:50.268370Z 2021-12-21T21:58:50.264009Z 2021-12-21T21:58:50.171365Z 2021-12-21T21:58:49.931923Z 2021-12-21T21:58:49.921541Z 2021-12-21T21:58:49.827881Z 2021-12-21T21:58:49.802912Z 2021-12-21T22:03:10.283315Z 2021-12-21T22:03:09.194623Z 2021-12-21T22:03:08.670940Z 2021-12-21T22:03:08.620624Z 2021-12-21T22:03:08.495154Z 2021-12-21T22:03:08.182003Z 2021-12-21T22:03:07.927433Z 2021-12-21T22:03:07.840002Z 2021-12-21T22:03:07.748592Z 2021-12-21T22:03:07.740046Z 2021-12-21T22:03:07.396316Z 2021-12-21T22:03:07.321346Z 2021-12-21T22:03:07.204120Z 2021-12-21T22:03:07.017663Z 2021-12-21T22:03:06.888410Z 2021-12-21T22:03:06.698670Z 2021-12-21T22:03:06.652123Z 2021-12-21T22:03:06.542821Z 2021-12-21T22:03:06.485668Z 2021-12-21T22:03:06.169843Z 2021-12-21T22:03:06.128002Z 2021-12-21T22:03:06.023079Z 2021-12-21T22:03:05.914221Z 2021-12-21T22:03:05.869753Z 2021-12-21T22:03:05.800720Z 2021-12-21T22:03:05.703941Z 2021-12-21T22:03:05.591922Z 2021-12-21T22:03:05.491626Z 2021-12-21T22:03:05.327053Z 2021-12-21T22:03:05.170582Z 2021-12-21T22:03:04.725958Z 2021-12-21T22:03:04.407138Z 2021-12-21T22:03:04.148779Z 2021-12-21T22:03:03.910990Z 2021-12-21T22:03:03.724465Z 2021-12-21T22:03:03.550415Z 2021-12-21T22:03:03.548101Z 2021-12-21T22:03:03.435517Z 2021-12-21T22:03:03.260617Z 2021-12-21T22:03:03.119305Z 2021-12-21T22:03:03.105368Z 2021-12-21T22:03:02.927484Z 2021-12-21T22:03:02.917916Z 2021-12-21T22:03:02.816565Z 2021-12-21T22:03:02.794758Z 2021-12-21T22:03:02.759496Z 2021-12-21T22:03:02.658744Z 2021-12-21T22:03:02.270004Z 2021-12-21T22:03:02.132825Z 2021-12-21T22:03:02.116575Z 2021-12-21T22:03:01.765586Z 2021-12-21T22:03:01.686485Z 2021-12-21T22:03:01.626191Z 2021-12-21T22:03:01.484142Z 2021-12-21T22:03:01.447774Z 2021-12-21T22:03:01.436844Z 2021-12-21T22:03:01.211993Z 2021-12-21T22:03:01.001511Z 2021-12-21T22:03:00.977623Z 2021-12-21T22:03:00.560595Z 2021-12-21T22:03:00.527745Z 2021-12-21T22:03:00.444390Z 2021-12-21T22:03:00.399091Z 2021-12-21T22:03:00.374201Z 2021-12-21T22:03:00.316015Z 2021-12-21T22:03:00.274929Z 2021-12-21T22:03:00.129122Z 2021-12-21T22:03:00.107728Z 2021-12-21T22:03:00.128055Z 2021-12-21T22:03:00.127212Z 2021-12-21T22:03:00.103237Z 2021-12-21T22:03:00.033515Z 2021-12-21T22:02:59.919218Z 2021-12-21T22:02:59.721525Z 2021-12-21T22:02:59.612617Z 2021-12-21T22:02:59.500661Z 2021-12-21T22:02:59.372368Z 2021-12-21T22:02:59.276384Z 2021-12-21T22:02:59.144878Z 2021-12-21T22:02:59.046089Z 2021-12-21T22:02:59.027397Z 2021-12-21T22:02:59.015213Z 2021-12-21T22:02:58.595140Z 2021-12-21T22:02:58.431127Z 2021-12-21T22:02:58.420009Z 2021-12-21T22:02:58.233074Z 2021-12-21T22:02:58.188229Z 2021-12-21T22:02:57.943878Z 2021-12-21T22:02:57.868865Z 2021-12-21T22:02:57.788856Z 2021-12-21T22:02:57.723175Z 2021-12-21T22:02:57.630184Z 2021-12-21T22:02:57.608883Z 2021-12-21T22:02:57.519694Z 2021-12-21T22:02:57.513316Z 2021-12-21T22:02:57.252691Z 2021-12-21T22:02:57.252691Z 2021-12-21T22:02:57.133327Z 2021-12-21T22:02:57.053018Z 2021-12-21T22:02:57.048103Z", + "created_diff": 16.060060800000002, + "ended": "2021-12-21T22:04:39.552607Z", + "error_job_ids": " []", + "event_first": "2021-12-21T21:47:26.955655Z 2021-12-21T21:47:15.298629Z 2021-12-21T21:47:32.739768Z 2021-12-21T21:47:18.496358Z 2021-12-21T21:47:38.054225Z 2021-12-21T21:47:09.642572Z 2021-12-21T21:47:21.186226Z 2021-12-21T21:47:03.522449Z 2021-12-21T21:47:21.537838Z 2021-12-21T21:46:58.986935Z 2021-12-21T21:47:12.402598Z 2021-12-21T21:46:52.978550Z 2021-12-21T21:47:10.198357Z 2021-12-21T21:48:53.626478Z 2021-12-21T21:46:48.147521Z 2021-12-21T21:48:15.530422Z 2021-12-21T21:46:50.135685Z 2021-12-21T21:46:45.568352Z 2021-12-21T21:46:46.229423Z 2021-12-21T21:47:57.022332Z 2021-12-21T21:46:48.392360Z 2021-12-21T21:48:33.369395Z 2021-12-21T21:46:45.022387Z 2021-12-21T21:48:44.557238Z 2021-12-21T21:46:44.268192Z 2021-12-21T21:49:03.469890Z 2021-12-21T21:46:38.495372Z 2021-12-21T21:48:24.972816Z 2021-12-21T21:46:38.541035Z 2021-12-21T21:48:06.635771Z 2021-12-21T21:46:32.457630Z 2021-12-21T21:47:31.386335Z 2021-12-21T21:46:32.216924Z 2021-12-21T21:46:36.821663Z 2021-12-21T21:46:30.349258Z 2021-12-21T21:48:08.859100Z 2021-12-21T21:47:53.511182Z 2021-12-21T21:48:03.493471Z 2021-12-21T21:47:51.895539Z 2021-12-21T21:48:08.613108Z 2021-12-21T21:47:55.173986Z 2021-12-21T21:48:07.623616Z 2021-12-21T21:47:53.764076Z 2021-12-21T21:48:06.566157Z 2021-12-21T21:47:54.891929Z 2021-12-21T21:48:05.266509Z 2021-12-21T21:48:05.003672Z 2021-12-21T21:48:05.250429Z 2021-12-21T21:48:06.566157Z 2021-12-21T21:48:05.003672Z 2021-12-21T21:47:58.427567Z 2021-12-21T21:47:51.229193Z 2021-12-21T21:47:59.594262Z 2021-12-21T21:47:40.362314Z 2021-12-21T21:47:57.670704Z 2021-12-21T21:47:47.029214Z 2021-12-21T21:47:55.888865Z 2021-12-21T21:47:47.029214Z 2021-12-21T21:47:58.929480Z 2021-12-21T21:47:49.203071Z 2021-12-21T21:48:07.325309Z 2021-12-21T21:48:01.984526Z 2021-12-21T21:47:43.511626Z 2021-12-21T21:47:56.923142Z 2021-12-21T21:47:44.019463Z 2021-12-21T21:48:07.325309Z 2021-12-21T21:47:55.878418Z 2021-12-21T21:47:45.256745Z 2021-12-21T21:47:56.154940Z 2021-12-21T21:47:41.840060Z 2021-12-21T21:47:53.348277Z 2021-12-21T21:47:47.785803Z 2021-12-21T21:47:44.827420Z 2021-12-21T21:47:47.029214Z 2021-12-21T21:47:51.838404Z 2021-12-21T21:47:52.405345Z 2021-12-21T21:47:42.625734Z 2021-12-21T21:47:49.459284Z 2021-12-21T21:47:53.348277Z 2021-12-21T21:47:41.586656Z 2021-12-21T21:47:50.100517Z 2021-12-21T21:47:41.586647Z 2021-12-21T21:47:45.157485Z 2021-12-21T21:47:33.395156Z 2021-12-21T21:47:46.964475Z 2021-12-21T21:47:33.149164Z 2021-12-21T21:47:53.088483Z 2021-12-21T21:47:38.044422Z 2021-12-21T21:47:41.795687Z 2021-12-21T21:47:36.098442Z 2021-12-21T21:47:43.895018Z 2021-12-21T21:47:36.273177Z 2021-12-21T21:47:42.559351Z 2021-12-21T21:47:34.714549Z 2021-12-21T21:47:40.909553Z 2021-12-21T21:47:25.296290Z 2021-12-21T21:47:36.662483Z 2021-12-21T21:47:22.910411Z 2021-12-21T21:47:35.547534Z 2021-12-21T21:47:27.621192Z 2021-12-21T21:52:25.969069Z 2021-12-21T21:52:24.411496Z 2021-12-21T21:52:28.665494Z 2021-12-21T21:52:28.324538Z 2021-12-21T21:52:29.834377Z 2021-12-21T21:52:28.324538Z 2021-12-21T21:52:27.067083Z 2021-12-21T21:52:27.313452Z 2021-12-21T21:52:28.392551Z 2021-12-21T21:52:24.411496Z 2021-12-21T21:52:24.708069Z 2021-12-21T21:52:29.968880Z 2021-12-21T21:52:27.403397Z 2021-12-21T21:52:30.216776Z 2021-12-21T21:52:24.793974Z 2021-12-21T21:52:27.116497Z 2021-12-21T21:52:22.374975Z 2021-12-21T21:52:26.669486Z 2021-12-21T21:52:24.708069Z 2021-12-21T21:52:29.332666Z 2021-12-21T21:52:24.493373Z 2021-12-21T21:52:20.814537Z 2021-12-21T21:52:25.563195Z 2021-12-21T21:52:26.430539Z 2021-12-21T21:52:24.106220Z 2021-12-21T21:52:21.584184Z 2021-12-21T21:52:21.433157Z 2021-12-21T21:52:22.036715Z 2021-12-21T21:52:20.167453Z 2021-12-21T21:52:23.375822Z 2021-12-21T21:52:14.822079Z 2021-12-21T21:52:21.504528Z 2021-12-21T21:52:18.086066Z 2021-12-21T21:52:21.504528Z 2021-12-21T21:52:17.584082Z 2021-12-21T21:52:28.324538Z 2021-12-21T21:52:11.425608Z 2021-12-21T21:52:20.108018Z 2021-12-21T21:52:20.675332Z 2021-12-21T21:52:15.144355Z 2021-12-21T21:52:13.614377Z 2021-12-21T21:52:15.792645Z 2021-12-21T21:52:19.660273Z 2021-12-21T21:52:14.628435Z 2021-12-21T21:52:13.977906Z 2021-12-21T21:52:19.551473Z 2021-12-21T21:52:12.217575Z 2021-12-21T21:52:12.156054Z 2021-12-21T21:52:09.929263Z 2021-12-21T21:52:06.977192Z 2021-12-21T21:52:11.939393Z 2021-12-21T21:52:15.738444Z 2021-12-21T21:52:12.979842Z 2021-12-21T21:52:12.839153Z 2021-12-21T21:51:58.872680Z 2021-12-21T21:52:08.779598Z 2021-12-21T21:52:08.627332Z 2021-12-21T21:52:09.839483Z 2021-12-21T21:52:06.735935Z 2021-12-21T21:52:09.505448Z 2021-12-21T21:52:11.939393Z 2021-12-21T21:52:03.247479Z 2021-12-21T21:52:00.890535Z 2021-12-21T21:52:02.666064Z 2021-12-21T21:52:00.050281Z 2021-12-21T21:52:03.921776Z 2021-12-21T21:51:55.285970Z 2021-12-21T21:52:02.913525Z 2021-12-21T21:51:55.083660Z 2021-12-21T21:51:48.607032Z 2021-12-21T21:51:28.750539Z 2021-12-21T21:51:35.292830Z 2021-12-21T21:51:25.868724Z 2021-12-21T21:51:32.681441Z 2021-12-21T21:51:26.366985Z 2021-12-21T21:51:27.166351Z 2021-12-21T21:51:23.181872Z 2021-12-21T21:51:31.255898Z 2021-12-21T21:51:23.943271Z 2021-12-21T21:51:25.698778Z 2021-12-21T21:51:19.003363Z 2021-12-21T21:51:25.675447Z 2021-12-21T21:51:21.909939Z 2021-12-21T21:51:26.225082Z 2021-12-21T21:51:18.987272Z 2021-12-21T21:51:30.792713Z 2021-12-21T21:51:19.844238Z 2021-12-21T21:51:22.068217Z 2021-12-21T21:51:19.235170Z 2021-12-21T21:51:15.906543Z 2021-12-21T21:51:08.827126Z 2021-12-21T21:51:12.346095Z 2021-12-21T21:51:05.594229Z 2021-12-21T21:51:10.435461Z 2021-12-21T21:51:06.794639Z 2021-12-21T21:51:08.269307Z 2021-12-21T21:51:04.615409Z 2021-12-21T21:51:05.239077Z 2021-12-21T21:51:01.838011Z 2021-12-21T21:51:01.121588Z 2021-12-21T21:56:21.075181Z 2021-12-21T21:56:24.556690Z 2021-12-21T21:56:21.295057Z 2021-12-21T21:56:24.103628Z 2021-12-21T21:56:22.609599Z 2021-12-21T21:56:24.348489Z 2021-12-21T21:56:21.075181Z 2021-12-21T21:56:20.449681Z 2021-12-21T21:56:22.761133Z 2021-12-21T21:56:15.665400Z 2021-12-21T21:56:19.922903Z 2021-12-21T21:56:17.799093Z 2021-12-21T21:56:19.722427Z 2021-12-21T21:56:20.558361Z 2021-12-21T21:56:18.438955Z 2021-12-21T21:56:14.826304Z 2021-12-21T21:56:19.554885Z 2021-12-21T21:56:13.171958Z 2021-12-21T21:56:17.904821Z 2021-12-21T21:56:18.008459Z 2021-12-21T21:56:25.029024Z 2021-12-21T21:56:21.455767Z 2021-12-21T21:56:09.618933Z 2021-12-21T21:56:22.334705Z 2021-12-21T21:56:18.246503Z 2021-12-21T21:56:19.436595Z 2021-12-21T21:56:15.194106Z 2021-12-21T21:56:18.957010Z 2021-12-21T21:56:12.652153Z 2021-12-21T21:56:16.632258Z 2021-12-21T21:56:11.985542Z 2021-12-21T21:56:10.049551Z 2021-12-21T21:56:12.155151Z 2021-12-21T21:56:11.168431Z 2021-12-21T21:56:10.975313Z 2021-12-21T21:56:11.750087Z 2021-12-21T21:56:09.514944Z 2021-12-21T21:56:13.645747Z 2021-12-21T21:56:15.412198Z 2021-12-21T21:56:12.596124Z 2021-12-21T21:56:13.849553Z 2021-12-21T21:56:09.905066Z 2021-12-21T21:56:10.301517Z 2021-12-21T21:56:10.894706Z 2021-12-21T21:56:09.905066Z 2021-12-21T21:56:16.416791Z 2021-12-21T21:56:03.894463Z 2021-12-21T21:56:00.856867Z 2021-12-21T21:56:05.429444Z 2021-12-21T21:56:11.446320Z 2021-12-21T21:56:02.645923Z 2021-12-21T21:56:10.553351Z 2021-12-21T21:56:00.229076Z 2021-12-21T21:56:05.588591Z 2021-12-21T21:56:00.141355Z 2021-12-21T21:56:07.726730Z 2021-12-21T21:56:00.200224Z 2021-12-21T21:56:01.613458Z 2021-12-21T21:55:58.418622Z 2021-12-21T21:55:57.591567Z 2021-12-21T21:55:59.031222Z 2021-12-21T21:55:56.220050Z 2021-12-21T21:55:52.783164Z 2021-12-21T21:56:02.119465Z 2021-12-21T21:55:51.384079Z 2021-12-21T21:55:55.947360Z 2021-12-21T21:55:46.109801Z 2021-12-21T21:55:51.020781Z 2021-12-21T21:55:51.274563Z 2021-12-21T21:55:51.914316Z 2021-12-21T21:55:41.965399Z 2021-12-21T21:55:45.526089Z 2021-12-21T21:55:37.549994Z 2021-12-21T21:55:45.824503Z 2021-12-21T21:55:39.984064Z 2021-12-21T21:55:40.384178Z 2021-12-21T21:55:40.225211Z 2021-12-21T21:55:44.769448Z 2021-12-21T21:55:39.723612Z 2021-12-21T21:55:28.791240Z 2021-12-21T21:55:12.852324Z 2021-12-21T21:55:10.586676Z 2021-12-21T21:55:13.604824Z 2021-12-21T21:55:12.293435Z 2021-12-21T21:55:10.016825Z 2021-12-21T21:55:10.515363Z 2021-12-21T21:55:10.171921Z 2021-12-21T21:55:09.587748Z 2021-12-21T21:55:08.682288Z 2021-12-21T21:55:12.293435Z 2021-12-21T21:55:06.351216Z 2021-12-21T21:55:08.129487Z 2021-12-21T21:55:06.625428Z 2021-12-21T21:55:06.130048Z 2021-12-21T21:55:04.814519Z 2021-12-21T21:55:01.063562Z 2021-12-21T21:54:57.489254Z 2021-12-21T21:54:57.606171Z 2021-12-21T21:54:54.709826Z 2021-12-21T21:54:51.884548Z 2021-12-21T22:00:28.765242Z 2021-12-21T22:00:23.726217Z 2021-12-21T22:00:22.714061Z 2021-12-21T22:00:26.498430Z 2021-12-21T22:00:24.880428Z 2021-12-21T22:00:29.428525Z 2021-12-21T22:00:24.119618Z 2021-12-21T22:00:21.410486Z 2021-12-21T22:00:21.084156Z 2021-12-21T22:00:26.954945Z 2021-12-21T22:00:27.269522Z 2021-12-21T22:00:27.529598Z 2021-12-21T22:00:22.843252Z 2021-12-21T22:00:24.144519Z 2021-12-21T22:00:25.131370Z 2021-12-21T22:00:20.701150Z 2021-12-21T22:00:23.026580Z 2021-12-21T22:00:24.361286Z 2021-12-21T22:00:18.014649Z 2021-12-21T22:00:22.437220Z 2021-12-21T22:00:22.714061Z 2021-12-21T22:00:25.789789Z 2021-12-21T22:00:15.789332Z 2021-12-21T22:00:29.024142Z 2021-12-21T22:00:20.701219Z 2021-12-21T22:00:29.024142Z 2021-12-21T22:00:23.101090Z 2021-12-21T22:00:25.046553Z 2021-12-21T22:00:12.065611Z 2021-12-21T22:00:24.542347Z 2021-12-21T22:00:20.825318Z 2021-12-21T22:00:22.757284Z 2021-12-21T22:00:19.171226Z 2021-12-21T22:00:21.919803Z 2021-12-21T22:00:22.214200Z 2021-12-21T22:00:16.986361Z 2021-12-21T22:00:17.513437Z 2021-12-21T22:00:19.420820Z 2021-12-21T22:00:15.711735Z 2021-12-21T22:00:19.656664Z 2021-12-21T22:00:09.536193Z 2021-12-21T22:00:21.665549Z 2021-12-21T22:00:11.558581Z 2021-12-21T22:00:17.624405Z 2021-12-21T22:00:13.588070Z 2021-12-21T22:00:19.656664Z 2021-12-21T22:00:16.034174Z 2021-12-21T22:00:17.175836Z 2021-12-21T22:00:12.634068Z 2021-12-21T22:00:10.652512Z 2021-12-21T22:00:10.813313Z 2021-12-21T22:00:20.509527Z 2021-12-21T22:00:07.267385Z 2021-12-21T22:00:12.455344Z 2021-12-21T22:00:12.426729Z 2021-12-21T22:00:10.598445Z 2021-12-21T22:00:15.340502Z 2021-12-21T22:00:07.797077Z 2021-12-21T22:00:10.309832Z 2021-12-21T22:00:02.089204Z 2021-12-21T22:00:11.678237Z 2021-12-21T22:00:08.429071Z 2021-12-21T22:00:03.265150Z 2021-12-21T22:00:05.810393Z 2021-12-21T22:00:01.101138Z 2021-12-21T22:00:01.180832Z 2021-12-21T21:59:53.886832Z 2021-12-21T22:00:00.519595Z 2021-12-21T21:59:38.151530Z 2021-12-21T21:59:56.151438Z 2021-12-21T21:59:50.422263Z 2021-12-21T22:00:04.260423Z 2021-12-21T21:59:51.425970Z 2021-12-21T22:00:00.312497Z 2021-12-21T21:59:49.498140Z 2021-12-21T21:59:47.348706Z 2021-12-21T21:59:38.193505Z 2021-12-21T21:59:41.998107Z 2021-12-21T21:59:30.204210Z 2021-12-21T21:59:44.037400Z 2021-12-21T21:59:27.000356Z 2021-12-21T21:59:49.579687Z 2021-12-21T21:59:20.554138Z 2021-12-21T21:59:16.044443Z 2021-12-21T21:59:10.647459Z 2021-12-21T21:59:13.540678Z 2021-12-21T21:59:09.561191Z 2021-12-21T21:59:13.773524Z 2021-12-21T21:59:09.561191Z 2021-12-21T21:59:13.772509Z 2021-12-21T21:59:09.617181Z 2021-12-21T21:59:11.643774Z 2021-12-21T21:59:08.751223Z 2021-12-21T21:59:09.922453Z 2021-12-21T21:59:07.943722Z 2021-12-21T21:59:08.825247Z 2021-12-21T21:59:02.767084Z 2021-12-21T21:59:02.797769Z 2021-12-21T21:58:59.848868Z 2021-12-21T21:59:00.570857Z 2021-12-21T22:04:37.341840Z 2021-12-21T22:04:37.439114Z 2021-12-21T22:04:35.174179Z 2021-12-21T22:04:34.905329Z 2021-12-21T22:04:31.209362Z 2021-12-21T22:04:36.124029Z 2021-12-21T22:04:31.949388Z 2021-12-21T22:04:30.504436Z 2021-12-21T22:04:32.810935Z 2021-12-21T22:04:36.124029Z 2021-12-21T22:04:32.810935Z 2021-12-21T22:04:33.086638Z 2021-12-21T22:04:34.159390Z 2021-12-21T22:04:33.591174Z 2021-12-21T22:04:34.688928Z 2021-12-21T22:04:32.772024Z 2021-12-21T22:04:32.810935Z 2021-12-21T22:04:35.367942Z 2021-12-21T22:04:25.408892Z 2021-12-21T22:04:28.635537Z 2021-12-21T22:04:32.111509Z 2021-12-21T22:04:29.192194Z 2021-12-21T22:04:29.978454Z 2021-12-21T22:04:37.439114Z 2021-12-21T22:04:35.246740Z 2021-12-21T22:04:37.300408Z 2021-12-21T22:04:32.273623Z 2021-12-21T22:04:34.686875Z 2021-12-21T22:04:28.550431Z 2021-12-21T22:04:27.826086Z 2021-12-21T22:04:21.721679Z 2021-12-21T22:04:29.760086Z 2021-12-21T22:04:31.952527Z 2021-12-21T22:04:30.223138Z 2021-12-21T22:04:30.195087Z 2021-12-21T22:04:30.963484Z 2021-12-21T22:04:32.482061Z 2021-12-21T22:04:31.194847Z 2021-12-21T22:04:26.621678Z 2021-12-21T22:04:35.103429Z 2021-12-21T22:04:26.874627Z 2021-12-21T22:04:28.449210Z 2021-12-21T22:04:26.613945Z 2021-12-21T22:04:23.562557Z 2021-12-21T22:04:28.569056Z 2021-12-21T22:04:23.322225Z 2021-12-21T22:04:28.637064Z 2021-12-21T22:04:18.503164Z 2021-12-21T22:04:08.784096Z 2021-12-21T22:04:19.766426Z 2021-12-21T22:04:23.562557Z 2021-12-21T22:04:16.457633Z 2021-12-21T22:04:13.654306Z 2021-12-21T22:04:14.036353Z 2021-12-21T22:04:18.557210Z 2021-12-21T22:04:09.142250Z 2021-12-21T22:04:09.745369Z 2021-12-21T22:04:11.496407Z 2021-12-21T22:04:15.395499Z 2021-12-21T22:04:15.023534Z 2021-12-21T22:04:13.935488Z 2021-12-21T22:04:13.609798Z 2021-12-21T22:04:11.307202Z 2021-12-21T22:04:14.795424Z 2021-12-21T22:04:15.390336Z 2021-12-21T22:04:14.313249Z 2021-12-21T22:04:13.658399Z 2021-12-21T22:04:02.834529Z 2021-12-21T22:04:08.377013Z 2021-12-21T22:04:10.401359Z 2021-12-21T22:04:08.022556Z 2021-12-21T22:03:54.668434Z 2021-12-21T22:04:07.484368Z 2021-12-21T22:04:06.384592Z 2021-12-21T22:04:04.227825Z 2021-12-21T22:03:56.695650Z 2021-12-21T22:03:53.741455Z 2021-12-21T22:03:59.541483Z 2021-12-21T22:03:59.992628Z 2021-12-21T22:03:51.492765Z 2021-12-21T22:03:54.498994Z 2021-12-21T22:03:50.922176Z 2021-12-21T22:03:54.103681Z 2021-12-21T22:03:52.279606Z 2021-12-21T22:03:47.677493Z 2021-12-21T22:03:45.333207Z 2021-12-21T22:03:47.671590Z 2021-12-21T22:03:24.956659Z 2021-12-21T22:03:24.593000Z 2021-12-21T22:03:24.243632Z 2021-12-21T22:03:25.317534Z 2021-12-21T22:03:21.541617Z 2021-12-21T22:03:15.959315Z 2021-12-21T22:03:17.450760Z 2021-12-21T22:03:12.127504Z 2021-12-21T22:03:07.820038Z 2021-12-21T22:03:13.094930Z 2021-12-21T22:03:12.326499Z 2021-12-21T22:03:08.287073Z 2021-12-21T22:03:11.270457Z", + "event_last": "2021-12-21T21:47:30.787325Z 2021-12-21T21:47:18.473076Z 2021-12-21T21:47:35.830207Z 2021-12-21T21:47:22.067904Z 2021-12-21T21:47:42.408549Z 2021-12-21T21:47:13.176137Z 2021-12-21T21:47:25.835129Z 2021-12-21T21:47:07.860007Z 2021-12-21T21:47:25.516164Z 2021-12-21T21:47:01.810921Z 2021-12-21T21:47:16.809188Z 2021-12-21T21:46:54.571035Z 2021-12-21T21:47:12.474946Z 2021-12-21T21:48:53.972722Z 2021-12-21T21:46:49.504848Z 2021-12-21T21:48:16.907407Z 2021-12-21T21:46:51.297531Z 2021-12-21T21:46:47.169313Z 2021-12-21T21:46:48.148479Z 2021-12-21T21:47:57.672931Z 2021-12-21T21:46:49.601194Z 2021-12-21T21:48:33.993302Z 2021-12-21T21:46:46.215661Z 2021-12-21T21:48:45.102619Z 2021-12-21T21:46:45.275171Z 2021-12-21T21:49:03.864641Z 2021-12-21T21:46:40.339257Z 2021-12-21T21:48:25.384909Z 2021-12-21T21:46:40.339257Z 2021-12-21T21:48:09.319382Z 2021-12-21T21:46:36.701093Z 2021-12-21T21:47:35.156878Z 2021-12-21T21:46:33.748042Z 2021-12-21T21:46:37.262317Z 2021-12-21T21:46:31.821899Z 2021-12-21T21:48:11.492696Z 2021-12-21T21:47:54.642909Z 2021-12-21T21:48:05.627441Z 2021-12-21T21:47:53.486682Z 2021-12-21T21:48:09.720778Z 2021-12-21T21:47:56.244918Z 2021-12-21T21:48:09.295700Z 2021-12-21T21:47:54.955969Z 2021-12-21T21:48:08.597920Z 2021-12-21T21:47:55.886187Z 2021-12-21T21:48:07.334008Z 2021-12-21T21:48:06.807176Z 2021-12-21T21:48:07.272176Z 2021-12-21T21:48:08.413339Z 2021-12-21T21:48:07.268561Z 2021-12-21T21:48:02.806455Z 2021-12-21T21:47:53.069528Z 2021-12-21T21:48:01.928594Z 2021-12-21T21:47:44.119621Z 2021-12-21T21:48:00.262275Z 2021-12-21T21:47:48.821915Z 2021-12-21T21:48:00.250380Z 2021-12-21T21:47:49.337286Z 2021-12-21T21:48:02.012283Z 2021-12-21T21:47:51.668970Z 2021-12-21T21:48:09.574239Z 2021-12-21T21:48:04.722233Z 2021-12-21T21:47:46.920399Z 2021-12-21T21:48:00.937206Z 2021-12-21T21:47:46.861107Z 2021-12-21T21:48:08.885248Z 2021-12-21T21:47:59.537077Z 2021-12-21T21:47:47.911910Z 2021-12-21T21:47:59.531607Z 2021-12-21T21:47:44.948493Z 2021-12-21T21:47:56.985109Z 2021-12-21T21:47:49.578533Z 2021-12-21T21:47:49.191149Z 2021-12-21T21:47:49.801394Z 2021-12-21T21:47:55.209315Z 2021-12-21T21:47:55.473248Z 2021-12-21T21:47:46.574367Z 2021-12-21T21:47:51.668970Z 2021-12-21T21:47:56.254410Z 2021-12-21T21:47:45.179142Z 2021-12-21T21:47:52.456214Z 2021-12-21T21:47:45.236926Z 2021-12-21T21:47:49.875185Z 2021-12-21T21:47:36.382928Z 2021-12-21T21:47:51.399655Z 2021-12-21T21:47:36.891764Z 2021-12-21T21:47:56.985109Z 2021-12-21T21:47:41.691448Z 2021-12-21T21:47:45.817829Z 2021-12-21T21:47:39.143115Z 2021-12-21T21:47:48.594254Z 2021-12-21T21:47:39.643906Z 2021-12-21T21:47:47.083314Z 2021-12-21T21:47:38.749975Z 2021-12-21T21:47:45.226740Z 2021-12-21T21:47:28.959001Z 2021-12-21T21:47:41.371222Z 2021-12-21T21:47:26.426032Z 2021-12-21T21:47:38.422412Z 2021-12-21T21:47:31.023888Z 2021-12-21T21:52:27.982804Z 2021-12-21T21:52:27.428408Z 2021-12-21T21:52:29.856264Z 2021-12-21T21:52:30.170573Z 2021-12-21T21:52:30.307472Z 2021-12-21T21:52:30.128657Z 2021-12-21T21:52:29.101298Z 2021-12-21T21:52:29.285836Z 2021-12-21T21:52:29.682036Z 2021-12-21T21:52:26.796273Z 2021-12-21T21:52:26.488177Z 2021-12-21T21:52:31.476928Z 2021-12-21T21:52:29.337925Z 2021-12-21T21:52:31.469078Z 2021-12-21T21:52:27.025606Z 2021-12-21T21:52:29.377490Z 2021-12-21T21:52:25.403163Z 2021-12-21T21:52:28.841062Z 2021-12-21T21:52:26.995799Z 2021-12-21T21:52:31.081339Z 2021-12-21T21:52:26.673102Z 2021-12-21T21:52:24.021858Z 2021-12-21T21:52:27.604013Z 2021-12-21T21:52:28.422208Z 2021-12-21T21:52:26.406059Z 2021-12-21T21:52:24.962729Z 2021-12-21T21:52:24.229148Z 2021-12-21T21:52:25.232397Z 2021-12-21T21:52:22.778019Z 2021-12-21T21:52:26.201173Z 2021-12-21T21:52:18.168915Z 2021-12-21T21:52:24.989683Z 2021-12-21T21:52:21.199078Z 2021-12-21T21:52:24.894129Z 2021-12-21T21:52:20.791716Z 2021-12-21T21:52:30.603832Z 2021-12-21T21:52:14.321474Z 2021-12-21T21:52:23.160288Z 2021-12-21T21:52:23.866083Z 2021-12-21T21:52:18.086066Z 2021-12-21T21:52:17.080208Z 2021-12-21T21:52:19.225255Z 2021-12-21T21:52:23.944623Z 2021-12-21T21:52:18.520047Z 2021-12-21T21:52:16.417991Z 2021-12-21T21:52:22.538869Z 2021-12-21T21:52:15.718468Z 2021-12-21T21:52:15.400272Z 2021-12-21T21:52:13.329845Z 2021-12-21T21:52:10.833403Z 2021-12-21T21:52:14.936015Z 2021-12-21T21:52:19.085276Z 2021-12-21T21:52:16.657203Z 2021-12-21T21:52:15.703182Z 2021-12-21T21:52:01.913894Z 2021-12-21T21:52:12.768123Z 2021-12-21T21:52:12.022485Z 2021-12-21T21:52:13.452195Z 2021-12-21T21:52:09.518884Z 2021-12-21T21:52:11.847640Z 2021-12-21T21:52:15.694334Z 2021-12-21T21:52:07.194626Z 2021-12-21T21:52:04.208425Z 2021-12-21T21:52:06.416878Z 2021-12-21T21:52:04.109597Z 2021-12-21T21:52:06.929242Z 2021-12-21T21:51:58.528729Z 2021-12-21T21:52:06.458151Z 2021-12-21T21:51:58.127887Z 2021-12-21T21:51:52.896914Z 2021-12-21T21:51:30.991118Z 2021-12-21T21:51:38.372171Z 2021-12-21T21:51:27.487057Z 2021-12-21T21:51:35.476185Z 2021-12-21T21:51:27.487057Z 2021-12-21T21:51:29.162364Z 2021-12-21T21:51:24.834989Z 2021-12-21T21:51:33.591875Z 2021-12-21T21:51:25.239275Z 2021-12-21T21:51:27.632206Z 2021-12-21T21:51:20.420918Z 2021-12-21T21:51:27.499436Z 2021-12-21T21:51:23.056585Z 2021-12-21T21:51:28.276572Z 2021-12-21T21:51:20.280243Z 2021-12-21T21:51:32.608317Z 2021-12-21T21:51:21.333980Z 2021-12-21T21:51:23.589164Z 2021-12-21T21:51:20.503450Z 2021-12-21T21:51:17.088759Z 2021-12-21T21:51:09.628250Z 2021-12-21T21:51:13.991807Z 2021-12-21T21:51:06.883758Z 2021-12-21T21:51:12.011296Z 2021-12-21T21:51:08.861400Z 2021-12-21T21:51:09.836271Z 2021-12-21T21:51:05.887344Z 2021-12-21T21:51:06.368704Z 2021-12-21T21:51:03.057812Z 2021-12-21T21:51:05.239572Z 2021-12-21T21:56:22.575009Z 2021-12-21T21:56:25.473017Z 2021-12-21T21:56:22.711712Z 2021-12-21T21:56:25.269323Z 2021-12-21T21:56:23.313783Z 2021-12-21T21:56:25.269323Z 2021-12-21T21:56:22.550402Z 2021-12-21T21:56:22.777451Z 2021-12-21T21:56:23.417156Z 2021-12-21T21:56:19.098934Z 2021-12-21T21:56:22.851140Z 2021-12-21T21:56:20.508920Z 2021-12-21T21:56:22.991552Z 2021-12-21T21:56:22.389145Z 2021-12-21T21:56:20.646681Z 2021-12-21T21:56:17.397051Z 2021-12-21T21:56:21.809294Z 2021-12-21T21:56:16.125984Z 2021-12-21T21:56:20.296369Z 2021-12-21T21:56:20.246961Z 2021-12-21T21:56:25.697247Z 2021-12-21T21:56:23.602260Z 2021-12-21T21:56:13.171958Z 2021-12-21T21:56:23.778605Z 2021-12-21T21:56:20.122979Z 2021-12-21T21:56:22.925235Z 2021-12-21T21:56:19.100980Z 2021-12-21T21:56:21.805883Z 2021-12-21T21:56:15.731979Z 2021-12-21T21:56:19.394756Z 2021-12-21T21:56:14.775967Z 2021-12-21T21:56:13.222335Z 2021-12-21T21:56:14.775967Z 2021-12-21T21:56:14.611980Z 2021-12-21T21:56:14.149961Z 2021-12-21T21:56:15.573268Z 2021-12-21T21:56:12.373303Z 2021-12-21T21:56:16.396383Z 2021-12-21T21:56:18.239759Z 2021-12-21T21:56:16.054187Z 2021-12-21T21:56:16.699733Z 2021-12-21T21:56:12.362115Z 2021-12-21T21:56:12.683224Z 2021-12-21T21:56:13.431378Z 2021-12-21T21:56:13.160914Z 2021-12-21T21:56:19.238215Z 2021-12-21T21:56:07.641983Z 2021-12-21T21:56:04.383188Z 2021-12-21T21:56:08.784881Z 2021-12-21T21:56:15.225199Z 2021-12-21T21:56:06.951960Z 2021-12-21T21:56:14.084353Z 2021-12-21T21:56:02.783891Z 2021-12-21T21:56:08.941191Z 2021-12-21T21:56:03.434735Z 2021-12-21T21:56:09.916794Z 2021-12-21T21:56:04.228654Z 2021-12-21T21:56:04.918408Z 2021-12-21T21:56:02.448589Z 2021-12-21T21:56:00.955993Z 2021-12-21T21:56:01.245189Z 2021-12-21T21:55:59.647651Z 2021-12-21T21:55:56.452812Z 2021-12-21T21:56:05.395207Z 2021-12-21T21:55:55.345942Z 2021-12-21T21:56:00.112165Z 2021-12-21T21:55:51.006929Z 2021-12-21T21:55:54.603166Z 2021-12-21T21:55:56.079399Z 2021-12-21T21:55:56.086183Z 2021-12-21T21:55:46.631937Z 2021-12-21T21:55:49.506176Z 2021-12-21T21:55:41.512475Z 2021-12-21T21:55:49.455204Z 2021-12-21T21:55:44.857907Z 2021-12-21T21:55:45.081454Z 2021-12-21T21:55:44.439563Z 2021-12-21T21:55:48.673777Z 2021-12-21T21:55:43.720897Z 2021-12-21T21:55:32.908246Z 2021-12-21T21:55:14.879758Z 2021-12-21T21:55:12.790351Z 2021-12-21T21:55:15.316942Z 2021-12-21T21:55:14.448825Z 2021-12-21T21:55:12.857638Z 2021-12-21T21:55:12.267274Z 2021-12-21T21:55:11.483348Z 2021-12-21T21:55:11.626680Z 2021-12-21T21:55:10.171921Z 2021-12-21T21:55:14.482088Z 2021-12-21T21:55:07.838734Z 2021-12-21T21:55:09.678224Z 2021-12-21T21:55:07.509960Z 2021-12-21T21:55:07.819957Z 2021-12-21T21:55:06.095428Z 2021-12-21T21:55:02.301657Z 2021-12-21T21:55:00.331995Z 2021-12-21T21:54:58.968231Z 2021-12-21T21:54:57.489254Z 2021-12-21T21:54:54.786109Z 2021-12-21T22:00:30.600463Z 2021-12-21T22:00:26.805047Z 2021-12-21T22:00:25.362830Z 2021-12-21T22:00:29.165667Z 2021-12-21T22:00:27.509033Z 2021-12-21T22:00:30.883734Z 2021-12-21T22:00:26.304785Z 2021-12-21T22:00:23.847274Z 2021-12-21T22:00:24.575540Z 2021-12-21T22:00:29.697307Z 2021-12-21T22:00:30.353653Z 2021-12-21T22:00:29.633098Z 2021-12-21T22:00:25.069822Z 2021-12-21T22:00:27.524204Z 2021-12-21T22:00:26.436394Z 2021-12-21T22:00:23.856409Z 2021-12-21T22:00:26.240223Z 2021-12-21T22:00:26.732366Z 2021-12-21T22:00:22.119614Z 2021-12-21T22:00:26.459937Z 2021-12-21T22:00:25.667260Z 2021-12-21T22:00:28.386038Z 2021-12-21T22:00:19.121561Z 2021-12-21T22:00:30.600463Z 2021-12-21T22:00:24.090362Z 2021-12-21T22:00:30.679516Z 2021-12-21T22:00:26.398936Z 2021-12-21T22:00:27.916033Z 2021-12-21T22:00:15.303928Z 2021-12-21T22:00:27.524204Z 2021-12-21T22:00:24.722368Z 2021-12-21T22:00:26.604186Z 2021-12-21T22:00:23.013076Z 2021-12-21T22:00:25.111386Z 2021-12-21T22:00:24.843607Z 2021-12-21T22:00:20.656712Z 2021-12-21T22:00:22.416913Z 2021-12-21T22:00:22.865327Z 2021-12-21T22:00:19.710882Z 2021-12-21T22:00:23.043176Z 2021-12-21T22:00:12.756224Z 2021-12-21T22:00:24.721505Z 2021-12-21T22:00:15.289137Z 2021-12-21T22:00:22.080192Z 2021-12-21T22:00:17.431987Z 2021-12-21T22:00:23.623507Z 2021-12-21T22:00:19.897427Z 2021-12-21T22:00:20.902020Z 2021-12-21T22:00:15.711735Z 2021-12-21T22:00:13.575754Z 2021-12-21T22:00:13.360994Z 2021-12-21T22:00:24.282229Z 2021-12-21T22:00:09.845927Z 2021-12-21T22:00:15.597518Z 2021-12-21T22:00:15.379931Z 2021-12-21T22:00:15.206249Z 2021-12-21T22:00:19.043534Z 2021-12-21T22:00:11.233177Z 2021-12-21T22:00:13.470939Z 2021-12-21T22:00:05.389955Z 2021-12-21T22:00:15.090210Z 2021-12-21T22:00:12.488562Z 2021-12-21T22:00:07.080955Z 2021-12-21T22:00:09.768707Z 2021-12-21T22:00:03.963915Z 2021-12-21T22:00:05.015212Z 2021-12-21T21:59:56.516327Z 2021-12-21T22:00:04.856252Z 2021-12-21T21:59:43.073158Z 2021-12-21T21:59:59.580202Z 2021-12-21T21:59:54.137310Z 2021-12-21T22:00:07.676716Z 2021-12-21T21:59:55.204312Z 2021-12-21T22:00:04.209217Z 2021-12-21T21:59:52.814349Z 2021-12-21T21:59:51.464183Z 2021-12-21T21:59:42.472354Z 2021-12-21T21:59:46.104340Z 2021-12-21T21:59:33.800126Z 2021-12-21T21:59:48.606290Z 2021-12-21T21:59:30.882592Z 2021-12-21T21:59:53.480198Z 2021-12-21T21:59:22.383893Z 2021-12-21T21:59:17.734199Z 2021-12-21T21:59:11.025548Z 2021-12-21T21:59:14.994646Z 2021-12-21T21:59:10.338056Z 2021-12-21T21:59:15.601597Z 2021-12-21T21:59:10.082449Z 2021-12-21T21:59:15.432304Z 2021-12-21T21:59:10.300371Z 2021-12-21T21:59:12.910331Z 2021-12-21T21:59:09.471514Z 2021-12-21T21:59:10.879343Z 2021-12-21T21:59:08.641317Z 2021-12-21T21:59:09.988799Z 2021-12-21T21:59:03.760413Z 2021-12-21T21:59:03.855183Z 2021-12-21T21:59:00.632405Z 2021-12-21T21:59:01.968546Z 2021-12-21T22:04:39.552607Z 2021-12-21T22:04:39.092646Z 2021-12-21T22:04:36.957646Z 2021-12-21T22:04:36.939881Z 2021-12-21T22:04:33.518160Z 2021-12-21T22:04:38.242411Z 2021-12-21T22:04:35.302603Z 2021-12-21T22:04:33.297959Z 2021-12-21T22:04:36.015859Z 2021-12-21T22:04:39.255259Z 2021-12-21T22:04:35.082586Z 2021-12-21T22:04:36.191111Z 2021-12-21T22:04:36.273207Z 2021-12-21T22:04:36.494904Z 2021-12-21T22:04:37.341840Z 2021-12-21T22:04:35.573998Z 2021-12-21T22:04:35.594167Z 2021-12-21T22:04:37.615217Z 2021-12-21T22:04:27.931600Z 2021-12-21T22:04:31.498365Z 2021-12-21T22:04:35.443310Z 2021-12-21T22:04:32.327285Z 2021-12-21T22:04:32.807452Z 2021-12-21T22:04:39.448621Z 2021-12-21T22:04:37.574617Z 2021-12-21T22:04:39.293685Z 2021-12-21T22:04:35.664832Z 2021-12-21T22:04:37.962223Z 2021-12-21T22:04:31.315042Z 2021-12-21T22:04:30.617305Z 2021-12-21T22:04:25.049456Z 2021-12-21T22:04:32.415459Z 2021-12-21T22:04:34.601452Z 2021-12-21T22:04:31.876909Z 2021-12-21T22:04:32.785328Z 2021-12-21T22:04:34.145911Z 2021-12-21T22:04:36.097739Z 2021-12-21T22:04:33.953465Z 2021-12-21T22:04:29.768335Z 2021-12-21T22:04:37.541688Z 2021-12-21T22:04:30.167170Z 2021-12-21T22:04:30.969926Z 2021-12-21T22:04:28.405210Z 2021-12-21T22:04:26.752955Z 2021-12-21T22:04:31.587214Z 2021-12-21T22:04:26.085755Z 2021-12-21T22:04:31.101944Z 2021-12-21T22:04:22.551675Z 2021-12-21T22:04:12.920920Z 2021-12-21T22:04:23.692230Z 2021-12-21T22:04:27.063894Z 2021-12-21T22:04:20.339221Z 2021-12-21T22:04:17.384284Z 2021-12-21T22:04:18.018292Z 2021-12-21T22:04:22.351830Z 2021-12-21T22:04:13.365190Z 2021-12-21T22:04:14.266040Z 2021-12-21T22:04:15.835270Z 2021-12-21T22:04:19.036920Z 2021-12-21T22:04:18.885200Z 2021-12-21T22:04:17.596967Z 2021-12-21T22:04:18.503164Z 2021-12-21T22:04:16.160248Z 2021-12-21T22:04:19.762195Z 2021-12-21T22:04:19.303091Z 2021-12-21T22:04:18.209635Z 2021-12-21T22:04:17.109988Z 2021-12-21T22:04:07.478250Z 2021-12-21T22:04:12.494181Z 2021-12-21T22:04:14.476901Z 2021-12-21T22:04:11.562898Z 2021-12-21T22:03:58.799457Z 2021-12-21T22:04:11.348919Z 2021-12-21T22:04:10.561760Z 2021-12-21T22:04:08.032935Z 2021-12-21T22:04:02.233193Z 2021-12-21T22:03:57.935052Z 2021-12-21T22:04:03.429198Z 2021-12-21T22:04:04.252140Z 2021-12-21T22:03:56.020211Z 2021-12-21T22:03:58.148885Z 2021-12-21T22:03:55.795159Z 2021-12-21T22:03:58.324914Z 2021-12-21T22:03:57.734153Z 2021-12-21T22:03:52.080938Z 2021-12-21T22:03:50.358329Z 2021-12-21T22:03:52.434447Z 2021-12-21T22:03:27.742176Z 2021-12-21T22:03:26.589907Z 2021-12-21T22:03:27.043655Z 2021-12-21T22:03:27.759947Z 2021-12-21T22:03:24.294573Z 2021-12-21T22:03:17.799155Z 2021-12-21T22:03:19.808204Z 2021-12-21T22:03:13.283945Z 2021-12-21T22:03:08.954139Z 2021-12-21T22:03:14.430220Z 2021-12-21T22:03:13.494251Z 2021-12-21T22:03:09.349316Z 2021-12-21T22:03:12.342703Z", + "failed_job_ids": " []", + "finished": "2021-12-21T21:47:36.502144Z 2021-12-21T21:47:24.421813Z 2021-12-21T21:47:39.765004Z 2021-12-21T21:47:26.954551Z 2021-12-21T21:47:47.049074Z 2021-12-21T21:47:18.785351Z 2021-12-21T21:47:30.872636Z 2021-12-21T21:47:13.109199Z 2021-12-21T21:47:31.589161Z 2021-12-21T21:47:08.272781Z 2021-12-21T21:47:21.918436Z 2021-12-21T21:46:59.494412Z 2021-12-21T21:47:18.550142Z 2021-12-21T21:48:57.953884Z 2021-12-21T21:46:53.143160Z 2021-12-21T21:48:19.606371Z 2021-12-21T21:46:56.461422Z 2021-12-21T21:46:49.933887Z 2021-12-21T21:46:51.342829Z 2021-12-21T21:48:00.894489Z 2021-12-21T21:46:54.116024Z 2021-12-21T21:48:37.689652Z 2021-12-21T21:46:50.529599Z 2021-12-21T21:48:48.889938Z 2021-12-21T21:46:49.986056Z 2021-12-21T21:49:08.075904Z 2021-12-21T21:46:44.317445Z 2021-12-21T21:48:29.309968Z 2021-12-21T21:46:44.688516Z 2021-12-21T21:48:11.019752Z 2021-12-21T21:46:38.913455Z 2021-12-21T21:47:40.069179Z 2021-12-21T21:46:38.631867Z 2021-12-21T21:46:41.203332Z 2021-12-21T21:46:35.642932Z 2021-12-21T21:48:13.530086Z 2021-12-21T21:47:58.491065Z 2021-12-21T21:48:10.410749Z 2021-12-21T21:47:58.084029Z 2021-12-21T21:48:13.879987Z 2021-12-21T21:48:00.405106Z 2021-12-21T21:48:13.559620Z 2021-12-21T21:47:59.327120Z 2021-12-21T21:48:12.957056Z 2021-12-21T21:48:00.396938Z 2021-12-21T21:48:12.124856Z 2021-12-21T21:48:11.613546Z 2021-12-21T21:48:11.812031Z 2021-12-21T21:48:12.211888Z 2021-12-21T21:48:12.070923Z 2021-12-21T21:48:07.012939Z 2021-12-21T21:47:57.869616Z 2021-12-21T21:48:07.010107Z 2021-12-21T21:47:49.914722Z 2021-12-21T21:48:05.414118Z 2021-12-21T21:47:53.896681Z 2021-12-21T21:48:05.474843Z 2021-12-21T21:47:53.377049Z 2021-12-21T21:48:06.634678Z 2021-12-21T21:47:55.494572Z 2021-12-21T21:48:13.999845Z 2021-12-21T21:48:09.706157Z 2021-12-21T21:47:52.476660Z 2021-12-21T21:48:05.364642Z 2021-12-21T21:47:51.846909Z 2021-12-21T21:48:12.504772Z 2021-12-21T21:48:04.716255Z 2021-12-21T21:47:52.843156Z 2021-12-21T21:48:05.036066Z 2021-12-21T21:47:49.934933Z 2021-12-21T21:48:02.301670Z 2021-12-21T21:47:54.945201Z 2021-12-21T21:47:55.449549Z 2021-12-21T21:47:54.543205Z 2021-12-21T21:48:00.777461Z 2021-12-21T21:48:01.120160Z 2021-12-21T21:47:52.089392Z 2021-12-21T21:47:55.774339Z 2021-12-21T21:48:01.531447Z 2021-12-21T21:47:50.483916Z 2021-12-21T21:47:57.054308Z 2021-12-21T21:47:49.913293Z 2021-12-21T21:47:54.296529Z 2021-12-21T21:47:41.238997Z 2021-12-21T21:47:57.006528Z 2021-12-21T21:47:42.492284Z 2021-12-21T21:48:03.145945Z 2021-12-21T21:47:46.669750Z 2021-12-21T21:47:50.012316Z 2021-12-21T21:47:44.499288Z 2021-12-21T21:47:54.097277Z 2021-12-21T21:47:43.641719Z 2021-12-21T21:47:53.259833Z 2021-12-21T21:47:44.333926Z 2021-12-21T21:47:51.705394Z 2021-12-21T21:47:34.130659Z 2021-12-21T21:47:47.371267Z 2021-12-21T21:47:31.328643Z 2021-12-21T21:47:44.009038Z 2021-12-21T21:47:35.922925Z 2021-12-21T21:52:32.427987Z 2021-12-21T21:52:31.613754Z 2021-12-21T21:52:33.820612Z 2021-12-21T21:52:34.382111Z 2021-12-21T21:52:34.258895Z 2021-12-21T21:52:34.379266Z 2021-12-21T21:52:33.043961Z 2021-12-21T21:52:33.534166Z 2021-12-21T21:52:33.244524Z 2021-12-21T21:52:32.026819Z 2021-12-21T21:52:31.024421Z 2021-12-21T21:52:34.876518Z 2021-12-21T21:52:33.768543Z 2021-12-21T21:52:35.239091Z 2021-12-21T21:52:31.472227Z 2021-12-21T21:52:33.579514Z 2021-12-21T21:52:29.659299Z 2021-12-21T21:52:32.635849Z 2021-12-21T21:52:32.476822Z 2021-12-21T21:52:34.628117Z 2021-12-21T21:52:31.100212Z 2021-12-21T21:52:28.889104Z 2021-12-21T21:52:32.282439Z 2021-12-21T21:52:32.466854Z 2021-12-21T21:52:30.349568Z 2021-12-21T21:52:29.038072Z 2021-12-21T21:52:28.132870Z 2021-12-21T21:52:30.203639Z 2021-12-21T21:52:28.580416Z 2021-12-21T21:52:30.872191Z 2021-12-21T21:52:23.431837Z 2021-12-21T21:52:29.239397Z 2021-12-21T21:52:26.555337Z 2021-12-21T21:52:30.473714Z 2021-12-21T21:52:26.886488Z 2021-12-21T21:52:34.875837Z 2021-12-21T21:52:20.202328Z 2021-12-21T21:52:29.814145Z 2021-12-21T21:52:28.915155Z 2021-12-21T21:52:23.215062Z 2021-12-21T21:52:23.068531Z 2021-12-21T21:52:24.197077Z 2021-12-21T21:52:28.264928Z 2021-12-21T21:52:24.003260Z 2021-12-21T21:52:22.483669Z 2021-12-21T21:52:27.295359Z 2021-12-21T21:52:19.772006Z 2021-12-21T21:52:19.948942Z 2021-12-21T21:52:18.227404Z 2021-12-21T21:52:16.578403Z 2021-12-21T21:52:21.071584Z 2021-12-21T21:52:23.897140Z 2021-12-21T21:52:21.836948Z 2021-12-21T21:52:21.948646Z 2021-12-21T21:52:07.315784Z 2021-12-21T21:52:18.286081Z 2021-12-21T21:52:17.995842Z 2021-12-21T21:52:18.518149Z 2021-12-21T21:52:15.016279Z 2021-12-21T21:52:17.727096Z 2021-12-21T21:52:20.561927Z 2021-12-21T21:52:12.940291Z 2021-12-21T21:52:09.819860Z 2021-12-21T21:52:11.369052Z 2021-12-21T21:52:09.289995Z 2021-12-21T21:52:12.710858Z 2021-12-21T21:52:03.830019Z 2021-12-21T21:52:11.251202Z 2021-12-21T21:52:02.877296Z 2021-12-21T21:51:56.654669Z 2021-12-21T21:51:34.959960Z 2021-12-21T21:51:42.920180Z 2021-12-21T21:51:32.349787Z 2021-12-21T21:51:40.488780Z 2021-12-21T21:51:32.321494Z 2021-12-21T21:51:34.417077Z 2021-12-21T21:51:29.123353Z 2021-12-21T21:51:37.373173Z 2021-12-21T21:51:29.833218Z 2021-12-21T21:51:32.442211Z 2021-12-21T21:51:25.119130Z 2021-12-21T21:51:33.154275Z 2021-12-21T21:51:27.674188Z 2021-12-21T21:51:33.150918Z 2021-12-21T21:51:25.061070Z 2021-12-21T21:51:36.778018Z 2021-12-21T21:51:25.028718Z 2021-12-21T21:51:28.214604Z 2021-12-21T21:51:25.278044Z 2021-12-21T21:51:21.668733Z 2021-12-21T21:51:14.171672Z 2021-12-21T21:51:17.946883Z 2021-12-21T21:51:10.467755Z 2021-12-21T21:51:15.682104Z 2021-12-21T21:51:12.013430Z 2021-12-21T21:51:13.773450Z 2021-12-21T21:51:10.387336Z 2021-12-21T21:51:10.981093Z 2021-12-21T21:51:06.573613Z 2021-12-21T21:51:07.152910Z 2021-12-21T21:56:26.227253Z 2021-12-21T21:56:29.254098Z 2021-12-21T21:56:27.160324Z 2021-12-21T21:56:28.973903Z 2021-12-21T21:56:27.147558Z 2021-12-21T21:56:28.580092Z 2021-12-21T21:56:27.148153Z 2021-12-21T21:56:26.560390Z 2021-12-21T21:56:26.936312Z 2021-12-21T21:56:23.552023Z 2021-12-21T21:56:27.945241Z 2021-12-21T21:56:24.631017Z 2021-12-21T21:56:26.608129Z 2021-12-21T21:56:26.158505Z 2021-12-21T21:56:25.235149Z 2021-12-21T21:56:21.985299Z 2021-12-21T21:56:25.860046Z 2021-12-21T21:56:21.195257Z 2021-12-21T21:56:24.495735Z 2021-12-21T21:56:24.663118Z 2021-12-21T21:56:29.135539Z 2021-12-21T21:56:28.035472Z 2021-12-21T21:56:18.065183Z 2021-12-21T21:56:27.575005Z 2021-12-21T21:56:24.497456Z 2021-12-21T21:56:27.325689Z 2021-12-21T21:56:23.551273Z 2021-12-21T21:56:26.111130Z 2021-12-21T21:56:20.938763Z 2021-12-21T21:56:24.434040Z 2021-12-21T21:56:19.530886Z 2021-12-21T21:56:18.765172Z 2021-12-21T21:56:20.381824Z 2021-12-21T21:56:20.445954Z 2021-12-21T21:56:19.988706Z 2021-12-21T21:56:19.940254Z 2021-12-21T21:56:17.022593Z 2021-12-21T21:56:21.331152Z 2021-12-21T21:56:23.735400Z 2021-12-21T21:56:20.792250Z 2021-12-21T21:56:22.022904Z 2021-12-21T21:56:16.670120Z 2021-12-21T21:56:17.474157Z 2021-12-21T21:56:19.674164Z 2021-12-21T21:56:18.831159Z 2021-12-21T21:56:23.324116Z 2021-12-21T21:56:11.706788Z 2021-12-21T21:56:10.615406Z 2021-12-21T21:56:14.486919Z 2021-12-21T21:56:20.304289Z 2021-12-21T21:56:11.775737Z 2021-12-21T21:56:20.171077Z 2021-12-21T21:56:07.938773Z 2021-12-21T21:56:13.495248Z 2021-12-21T21:56:08.115664Z 2021-12-21T21:56:13.636169Z 2021-12-21T21:56:09.869852Z 2021-12-21T21:56:10.552592Z 2021-12-21T21:56:08.351350Z 2021-12-21T21:56:06.070150Z 2021-12-21T21:56:05.857582Z 2021-12-21T21:56:04.940325Z 2021-12-21T21:56:02.572656Z 2021-12-21T21:56:11.439557Z 2021-12-21T21:56:00.957067Z 2021-12-21T21:56:04.522326Z 2021-12-21T21:55:57.261837Z 2021-12-21T21:56:00.499798Z 2021-12-21T21:56:01.102962Z 2021-12-21T21:56:01.392130Z 2021-12-21T21:55:52.221936Z 2021-12-21T21:55:55.454098Z 2021-12-21T21:55:47.145913Z 2021-12-21T21:55:54.388337Z 2021-12-21T21:55:49.167210Z 2021-12-21T21:55:51.706907Z 2021-12-21T21:55:49.365930Z 2021-12-21T21:55:54.386447Z 2021-12-21T21:55:48.969918Z 2021-12-21T21:55:38.238010Z 2021-12-21T21:55:20.302790Z 2021-12-21T21:55:17.397164Z 2021-12-21T21:55:19.292823Z 2021-12-21T21:55:18.874209Z 2021-12-21T21:55:16.373683Z 2021-12-21T21:55:16.754449Z 2021-12-21T21:55:15.612822Z 2021-12-21T21:55:15.716334Z 2021-12-21T21:55:14.613081Z 2021-12-21T21:55:19.827116Z 2021-12-21T21:55:13.013360Z 2021-12-21T21:55:13.779221Z 2021-12-21T21:55:12.194808Z 2021-12-21T21:55:12.284215Z 2021-12-21T21:55:09.883128Z 2021-12-21T21:55:06.643290Z 2021-12-21T21:55:02.644060Z 2021-12-21T21:55:02.571065Z 2021-12-21T21:54:59.766165Z 2021-12-21T21:54:57.163659Z 2021-12-21T22:00:34.423987Z 2021-12-21T22:00:30.771746Z 2021-12-21T22:00:30.471137Z 2021-12-21T22:00:33.336910Z 2021-12-21T22:00:31.248613Z 2021-12-21T22:00:35.374759Z 2021-12-21T22:00:31.299957Z 2021-12-21T22:00:29.648249Z 2021-12-21T22:00:30.564147Z 2021-12-21T22:00:33.933613Z 2021-12-21T22:00:32.143961Z 2021-12-21T22:00:34.365275Z 2021-12-21T22:00:29.354268Z 2021-12-21T22:00:32.497057Z 2021-12-21T22:00:30.869526Z 2021-12-21T22:00:28.995634Z 2021-12-21T22:00:30.055786Z 2021-12-21T22:00:31.622053Z 2021-12-21T22:00:27.295637Z 2021-12-21T22:00:30.697895Z 2021-12-21T22:00:30.038888Z 2021-12-21T22:00:33.033701Z 2021-12-21T22:00:25.112113Z 2021-12-21T22:00:34.461570Z 2021-12-21T22:00:28.644860Z 2021-12-21T22:00:34.774262Z 2021-12-21T22:00:30.519632Z 2021-12-21T22:00:32.466517Z 2021-12-21T22:00:21.885361Z 2021-12-21T22:00:32.227472Z 2021-12-21T22:00:29.264901Z 2021-12-21T22:00:30.944436Z 2021-12-21T22:00:26.879646Z 2021-12-21T22:00:29.057096Z 2021-12-21T22:00:29.410448Z 2021-12-21T22:00:26.420415Z 2021-12-21T22:00:26.889198Z 2021-12-21T22:00:27.261881Z 2021-12-21T22:00:25.340172Z 2021-12-21T22:00:28.152408Z 2021-12-21T22:00:17.937709Z 2021-12-21T22:00:30.888994Z 2021-12-21T22:00:21.363675Z 2021-12-21T22:00:27.585041Z 2021-12-21T22:00:22.129857Z 2021-12-21T22:00:28.898625Z 2021-12-21T22:00:25.227409Z 2021-12-21T22:00:24.970396Z 2021-12-21T22:00:20.126332Z 2021-12-21T22:00:19.448272Z 2021-12-21T22:00:18.391084Z 2021-12-21T22:00:28.983043Z 2021-12-21T22:00:16.874237Z 2021-12-21T22:00:21.079007Z 2021-12-21T22:00:21.276912Z 2021-12-21T22:00:21.344228Z 2021-12-21T22:00:23.646004Z 2021-12-21T22:00:15.652591Z 2021-12-21T22:00:20.014902Z 2021-12-21T22:00:11.078926Z 2021-12-21T22:00:19.870404Z 2021-12-21T22:00:18.680025Z 2021-12-21T22:00:11.512754Z 2021-12-21T22:00:14.433112Z 2021-12-21T22:00:10.158868Z 2021-12-21T22:00:09.669930Z 2021-12-21T22:00:02.532579Z 2021-12-21T22:00:11.221150Z 2021-12-21T21:59:48.177604Z 2021-12-21T22:00:05.096820Z 2021-12-21T22:00:00.010903Z 2021-12-21T22:00:13.111948Z 2021-12-21T22:00:00.785097Z 2021-12-21T22:00:09.405185Z 2021-12-21T21:59:57.599571Z 2021-12-21T21:59:55.683857Z 2021-12-21T21:59:48.542850Z 2021-12-21T21:59:52.239483Z 2021-12-21T21:59:38.754468Z 2021-12-21T21:59:55.154014Z 2021-12-21T21:59:37.354303Z 2021-12-21T21:59:58.019048Z 2021-12-21T21:59:26.803033Z 2021-12-21T21:59:22.709377Z 2021-12-21T21:59:14.386846Z 2021-12-21T21:59:19.170766Z 2021-12-21T21:59:13.988178Z 2021-12-21T21:59:20.260075Z 2021-12-21T21:59:13.524761Z 2021-12-21T21:59:20.747495Z 2021-12-21T21:59:14.257607Z 2021-12-21T21:59:17.782164Z 2021-12-21T21:59:13.444817Z 2021-12-21T21:59:15.228666Z 2021-12-21T21:59:12.342364Z 2021-12-21T21:59:14.784537Z 2021-12-21T21:59:07.795758Z 2021-12-21T21:59:07.936870Z 2021-12-21T21:59:04.465816Z 2021-12-21T21:59:06.060017Z 2021-12-21T22:04:42.356022Z 2021-12-21T22:04:42.872274Z 2021-12-21T22:04:41.188014Z 2021-12-21T22:04:41.380372Z 2021-12-21T22:04:38.165095Z 2021-12-21T22:04:41.993403Z 2021-12-21T22:04:40.615926Z 2021-12-21T22:04:37.623022Z 2021-12-21T22:04:40.797019Z 2021-12-21T22:04:43.405251Z 2021-12-21T22:04:38.781741Z 2021-12-21T22:04:39.665856Z 2021-12-21T22:04:41.182567Z 2021-12-21T22:04:41.369526Z 2021-12-21T22:04:41.035280Z 2021-12-21T22:04:40.116233Z 2021-12-21T22:04:40.892405Z 2021-12-21T22:04:41.681785Z 2021-12-21T22:04:33.590290Z 2021-12-21T22:04:36.376140Z 2021-12-21T22:04:41.162315Z 2021-12-21T22:04:37.572619Z 2021-12-21T22:04:37.725091Z 2021-12-21T22:04:43.298948Z 2021-12-21T22:04:41.185615Z 2021-12-21T22:04:43.637626Z 2021-12-21T22:04:41.227987Z 2021-12-21T22:04:42.209777Z 2021-12-21T22:04:37.332500Z 2021-12-21T22:04:35.311287Z 2021-12-21T22:04:29.640100Z 2021-12-21T22:04:37.104425Z 2021-12-21T22:04:38.300897Z 2021-12-21T22:04:36.837511Z 2021-12-21T22:04:37.719503Z 2021-12-21T22:04:38.630013Z 2021-12-21T22:04:40.852993Z 2021-12-21T22:04:38.401481Z 2021-12-21T22:04:34.576153Z 2021-12-21T22:04:42.101618Z 2021-12-21T22:04:35.264104Z 2021-12-21T22:04:35.655811Z 2021-12-21T22:04:33.108546Z 2021-12-21T22:04:32.103877Z 2021-12-21T22:04:36.199833Z 2021-12-21T22:04:29.850244Z 2021-12-21T22:04:35.043324Z 2021-12-21T22:04:26.409219Z 2021-12-21T22:04:17.537906Z 2021-12-21T22:04:28.353031Z 2021-12-21T22:04:32.880898Z 2021-12-21T22:04:26.055996Z 2021-12-21T22:04:23.742033Z 2021-12-21T22:04:23.503052Z 2021-12-21T22:04:26.952661Z 2021-12-21T22:04:19.448307Z 2021-12-21T22:04:19.536768Z 2021-12-21T22:04:21.692462Z 2021-12-21T22:04:25.218535Z 2021-12-21T22:04:24.720929Z 2021-12-21T22:04:23.912793Z 2021-12-21T22:04:23.495052Z 2021-12-21T22:04:22.510587Z 2021-12-21T22:04:24.939066Z 2021-12-21T22:04:25.118099Z 2021-12-21T22:04:23.324951Z 2021-12-21T22:04:21.678609Z 2021-12-21T22:04:13.124524Z 2021-12-21T22:04:16.143272Z 2021-12-21T22:04:20.569803Z 2021-12-21T22:04:17.669857Z 2021-12-21T22:04:04.925928Z 2021-12-21T22:04:16.972926Z 2021-12-21T22:04:16.822472Z 2021-12-21T22:04:11.989744Z 2021-12-21T22:04:08.513096Z 2021-12-21T22:04:04.996803Z 2021-12-21T22:04:09.143764Z 2021-12-21T22:04:09.057881Z 2021-12-21T22:04:01.667954Z 2021-12-21T22:04:02.451545Z 2021-12-21T22:04:02.076244Z 2021-12-21T22:04:04.988608Z 2021-12-21T22:04:03.080957Z 2021-12-21T22:03:58.894688Z 2021-12-21T22:03:57.045161Z 2021-12-21T22:03:57.579924Z 2021-12-21T22:03:32.419161Z 2021-12-21T22:03:31.294244Z 2021-12-21T22:03:32.035928Z 2021-12-21T22:03:32.708453Z 2021-12-21T22:03:28.657073Z 2021-12-21T22:03:22.638879Z 2021-12-21T22:03:24.212527Z 2021-12-21T22:03:17.144059Z 2021-12-21T22:03:12.707653Z 2021-12-21T22:03:19.189072Z 2021-12-21T22:03:18.113534Z 2021-12-21T22:03:13.302778Z 2021-12-21T22:03:16.337579Z", + "finished_diff": 103.00556100000001, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 166.548702, + "mean": 49.778573495999986, + "min": 12.152251 + }, + "jobs_events_duration": { + "max": 5.537543, + "mean": 2.8146102560000013, + "min": 0.346244 + }, + "jobs_events_lag": { + "max": -1.70037, + "mean": -4.789891438000004, + "min": -7.061751 + }, + "jobs_waiting": { + "max": 45.563849, + "mean": 24.611729483999987, + "min": 0.665763 + }, + "modified": "2021-12-21T21:46:40.818150Z 2021-12-21T21:46:40.742816Z 2021-12-21T21:46:40.659615Z 2021-12-21T21:46:40.568024Z 2021-12-21T21:46:40.471143Z 2021-12-21T21:46:40.340309Z 2021-12-21T21:46:40.135060Z 2021-12-21T21:46:39.962882Z 2021-12-21T21:46:39.733933Z 2021-12-21T21:46:39.362507Z 2021-12-21T21:46:38.997775Z 2021-12-21T21:46:38.656639Z 2021-12-21T21:46:38.411292Z 2021-12-21T21:46:24.738113Z 2021-12-21T21:46:24.659128Z 2021-12-21T21:46:24.550418Z 2021-12-21T21:46:24.484432Z 2021-12-21T21:46:24.398226Z 2021-12-21T21:46:24.266327Z 2021-12-21T21:46:24.050588Z 2021-12-21T21:46:23.890023Z 2021-12-21T21:46:23.612789Z 2021-12-21T21:46:23.306981Z 2021-12-21T21:46:23.117140Z 2021-12-21T21:46:22.939331Z 2021-12-21T21:46:20.544253Z 2021-12-21T21:46:20.510001Z 2021-12-21T21:46:20.421877Z 2021-12-21T21:46:20.368299Z 2021-12-21T21:46:20.329512Z 2021-12-21T21:46:20.213533Z 2021-12-21T21:46:19.506882Z 2021-12-21T21:46:19.470612Z 2021-12-21T21:46:19.423315Z 2021-12-21T21:46:19.075875Z 2021-12-21T21:47:12.167378Z 2021-12-21T21:47:12.063908Z 2021-12-21T21:47:11.927913Z 2021-12-21T21:47:11.789034Z 2021-12-21T21:47:11.656403Z 2021-12-21T21:47:11.533864Z 2021-12-21T21:47:11.411282Z 2021-12-21T21:47:11.248289Z 2021-12-21T21:47:11.091947Z 2021-12-21T21:47:10.950237Z 2021-12-21T21:47:10.814060Z 2021-12-21T21:47:10.684210Z 2021-12-21T21:47:10.544050Z 2021-12-21T21:47:10.363227Z 2021-12-21T21:47:10.185954Z 2021-12-21T21:46:44.747321Z 2021-12-21T21:46:44.647504Z 2021-12-21T21:46:44.581142Z 2021-12-21T21:46:44.501106Z 2021-12-21T21:46:44.424622Z 2021-12-21T21:46:44.319852Z 2021-12-21T21:46:44.259170Z 2021-12-21T21:46:44.179852Z 2021-12-21T21:46:44.062132Z 2021-12-21T21:46:43.937089Z 2021-12-21T21:47:10.087925Z 2021-12-21T21:46:43.766768Z 2021-12-21T21:46:43.721981Z 2021-12-21T21:46:43.685379Z 2021-12-21T21:46:43.649447Z 2021-12-21T21:47:09.944137Z 2021-12-21T21:46:43.609878Z 2021-12-21T21:46:43.570570Z 2021-12-21T21:46:43.533459Z 2021-12-21T21:46:43.497329Z 2021-12-21T21:46:43.461917Z 2021-12-21T21:46:43.427381Z 2021-12-21T21:46:43.392835Z 2021-12-21T21:46:43.357421Z 2021-12-21T21:46:43.313044Z 2021-12-21T21:46:43.244152Z 2021-12-21T21:46:43.279591Z 2021-12-21T21:46:43.176308Z 2021-12-21T21:46:43.105149Z 2021-12-21T21:46:43.016281Z 2021-12-21T21:46:42.925092Z 2021-12-21T21:46:42.836087Z 2021-12-21T21:46:42.739265Z 2021-12-21T21:46:42.642745Z 2021-12-21T21:46:42.576248Z 2021-12-21T21:46:42.488956Z 2021-12-21T21:46:42.385130Z 2021-12-21T21:46:42.275068Z 2021-12-21T21:46:42.155190Z 2021-12-21T21:46:42.025496Z 2021-12-21T21:46:41.872474Z 2021-12-21T21:46:41.747146Z 2021-12-21T21:46:41.660273Z 2021-12-21T21:46:41.561195Z 2021-12-21T21:46:41.501749Z 2021-12-21T21:46:41.438187Z 2021-12-21T21:46:41.323193Z 2021-12-21T21:46:41.240075Z 2021-12-21T21:46:41.051476Z 2021-12-21T21:46:40.928144Z 2021-12-21T21:51:21.419224Z 2021-12-21T21:51:21.334347Z 2021-12-21T21:51:21.272058Z 2021-12-21T21:51:21.216437Z 2021-12-21T21:51:21.160008Z 2021-12-21T21:51:21.105020Z 2021-12-21T21:51:21.042336Z 2021-12-21T21:51:20.974170Z 2021-12-21T21:51:20.911079Z 2021-12-21T21:51:20.850016Z 2021-12-21T21:51:20.786135Z 2021-12-21T21:51:20.728177Z 2021-12-21T21:51:20.664274Z 2021-12-21T21:51:20.584121Z 2021-12-21T21:51:20.496259Z 2021-12-21T21:51:20.395069Z 2021-12-21T21:51:20.308051Z 2021-12-21T21:51:20.217041Z 2021-12-21T21:51:20.130409Z 2021-12-21T21:51:20.031105Z 2021-12-21T21:51:19.933389Z 2021-12-21T21:51:19.841042Z 2021-12-21T21:51:19.752247Z 2021-12-21T21:51:19.690260Z 2021-12-21T21:51:19.630992Z 2021-12-21T21:51:19.576209Z 2021-12-21T21:51:19.525735Z 2021-12-21T21:51:19.469000Z 2021-12-21T21:51:19.406050Z 2021-12-21T21:51:19.348292Z 2021-12-21T21:51:19.284886Z 2021-12-21T21:51:19.242084Z 2021-12-21T21:51:19.147014Z 2021-12-21T21:51:18.997225Z 2021-12-21T21:51:18.875095Z 2021-12-21T21:51:18.786274Z 2021-12-21T21:51:18.659059Z 2021-12-21T21:51:18.557223Z 2021-12-21T21:51:18.396515Z 2021-12-21T21:51:18.070257Z 2021-12-21T21:51:18.217128Z 2021-12-21T21:51:17.945156Z 2021-12-21T21:51:17.852339Z 2021-12-21T21:51:17.763148Z 2021-12-21T21:51:17.685123Z 2021-12-21T21:51:17.639133Z 2021-12-21T21:51:17.597998Z 2021-12-21T21:51:17.546205Z 2021-12-21T21:51:17.460875Z 2021-12-21T21:51:17.330253Z 2021-12-21T21:51:17.232290Z 2021-12-21T21:51:17.134430Z 2021-12-21T21:51:17.045956Z 2021-12-21T21:51:16.998922Z 2021-12-21T21:51:16.929850Z 2021-12-21T21:51:16.844974Z 2021-12-21T21:51:16.805030Z 2021-12-21T21:51:16.742801Z 2021-12-21T21:51:16.627390Z 2021-12-21T21:51:16.541060Z 2021-12-21T21:51:16.476276Z 2021-12-21T21:51:16.361131Z 2021-12-21T21:51:16.308189Z 2021-12-21T21:51:16.241591Z 2021-12-21T21:51:16.173848Z 2021-12-21T21:51:16.036442Z 2021-12-21T21:51:15.886189Z 2021-12-21T21:51:15.761588Z 2021-12-21T21:51:15.616831Z 2021-12-21T21:51:15.483082Z 2021-12-21T21:51:04.199755Z 2021-12-21T21:51:03.783283Z 2021-12-21T21:51:03.380232Z 2021-12-21T21:51:02.841654Z 2021-12-21T21:51:02.326470Z 2021-12-21T21:51:01.669890Z 2021-12-21T21:51:01.299362Z 2021-12-21T21:51:00.951560Z 2021-12-21T21:51:00.682396Z 2021-12-21T21:51:00.406919Z 2021-12-21T21:51:00.162395Z 2021-12-21T21:50:59.886083Z 2021-12-21T21:50:59.625162Z 2021-12-21T21:50:59.472267Z 2021-12-21T21:50:59.379247Z 2021-12-21T21:50:59.232467Z 2021-12-21T21:50:58.968207Z 2021-12-21T21:50:58.680900Z 2021-12-21T21:50:58.409726Z 2021-12-21T21:50:55.173020Z 2021-12-21T21:50:54.940820Z 2021-12-21T21:50:54.774213Z 2021-12-21T21:50:52.819275Z 2021-12-21T21:50:52.786793Z 2021-12-21T21:50:52.753893Z 2021-12-21T21:50:52.720268Z 2021-12-21T21:50:52.682715Z 2021-12-21T21:50:52.152915Z 2021-12-21T21:50:52.120075Z 2021-12-21T21:50:52.078497Z 2021-12-21T21:55:39.024136Z 2021-12-21T21:55:38.936676Z 2021-12-21T21:55:38.859992Z 2021-12-21T21:55:38.791994Z 2021-12-21T21:55:38.728382Z 2021-12-21T21:55:38.651872Z 2021-12-21T21:55:38.578432Z 2021-12-21T21:55:38.491934Z 2021-12-21T21:55:38.413027Z 2021-12-21T21:55:05.580296Z 2021-12-21T21:55:05.476089Z 2021-12-21T21:55:05.372200Z 2021-12-21T21:55:05.276153Z 2021-12-21T21:55:05.183063Z 2021-12-21T21:55:05.099315Z 2021-12-21T21:55:05.009251Z 2021-12-21T21:55:04.923943Z 2021-12-21T21:55:04.870062Z 2021-12-21T21:55:04.818167Z 2021-12-21T21:55:04.774368Z 2021-12-21T21:55:38.338250Z 2021-12-21T21:55:04.725145Z 2021-12-21T21:55:04.671230Z 2021-12-21T21:55:04.615310Z 2021-12-21T21:55:04.554264Z 2021-12-21T21:55:04.502270Z 2021-12-21T21:55:04.389151Z 2021-12-21T21:55:04.293174Z 2021-12-21T21:55:04.209147Z 2021-12-21T21:55:04.155996Z 2021-12-21T21:55:04.089351Z 2021-12-21T21:55:04.033733Z 2021-12-21T21:55:03.914181Z 2021-12-21T21:55:03.775360Z 2021-12-21T21:55:03.698926Z 2021-12-21T21:55:03.634502Z 2021-12-21T21:55:03.590241Z 2021-12-21T21:55:03.537010Z 2021-12-21T21:55:03.497542Z 2021-12-21T21:55:03.463468Z 2021-12-21T21:55:03.386239Z 2021-12-21T21:55:03.207293Z 2021-12-21T21:55:03.323092Z 2021-12-21T21:55:03.099288Z 2021-12-21T21:55:02.959183Z 2021-12-21T21:55:02.840270Z 2021-12-21T21:55:02.731933Z 2021-12-21T21:55:02.663089Z 2021-12-21T21:55:02.558810Z 2021-12-21T21:55:02.465109Z 2021-12-21T21:55:02.376090Z 2021-12-21T21:55:02.258479Z 2021-12-21T21:55:02.118289Z 2021-12-21T21:55:02.015404Z 2021-12-21T21:55:01.908260Z 2021-12-21T21:55:01.799271Z 2021-12-21T21:55:01.730333Z 2021-12-21T21:55:01.670277Z 2021-12-21T21:55:01.596543Z 2021-12-21T21:55:01.528240Z 2021-12-21T21:55:01.486287Z 2021-12-21T21:55:01.429475Z 2021-12-21T21:55:01.358077Z 2021-12-21T21:55:01.304691Z 2021-12-21T21:55:01.267090Z 2021-12-21T21:55:01.222310Z 2021-12-21T21:55:01.166671Z 2021-12-21T21:55:01.089552Z 2021-12-21T21:55:00.995202Z 2021-12-21T21:55:00.919124Z 2021-12-21T21:55:00.873646Z 2021-12-21T21:55:00.835743Z 2021-12-21T21:55:00.792179Z 2021-12-21T21:55:00.751443Z 2021-12-21T21:55:00.705150Z 2021-12-21T21:55:00.640698Z 2021-12-21T21:55:00.553220Z 2021-12-21T21:55:00.466100Z 2021-12-21T21:55:00.366001Z 2021-12-21T21:55:00.272563Z 2021-12-21T21:54:49.790080Z 2021-12-21T21:54:49.576562Z 2021-12-21T21:54:49.431848Z 2021-12-21T21:54:49.308168Z 2021-12-21T21:54:49.137797Z 2021-12-21T21:54:49.037843Z 2021-12-21T21:54:48.922888Z 2021-12-21T21:54:48.863219Z 2021-12-21T21:54:48.649565Z 2021-12-21T21:54:48.397220Z 2021-12-21T21:54:48.195464Z 2021-12-21T21:54:48.066659Z 2021-12-21T21:54:47.994656Z 2021-12-21T21:54:47.891497Z 2021-12-21T21:54:47.772477Z 2021-12-21T21:54:46.153233Z 2021-12-21T21:54:46.097005Z 2021-12-21T21:54:46.038439Z 2021-12-21T21:54:45.401866Z 2021-12-21T21:54:44.753411Z 2021-12-21T21:59:39.966833Z 2021-12-21T21:59:39.884291Z 2021-12-21T21:59:10.133287Z 2021-12-21T21:59:10.085212Z 2021-12-21T21:59:10.033622Z 2021-12-21T21:59:09.958775Z 2021-12-21T21:59:09.902078Z 2021-12-21T21:59:09.863311Z 2021-12-21T21:59:09.770660Z 2021-12-21T21:59:09.669019Z 2021-12-21T21:59:09.534176Z 2021-12-21T21:59:09.408316Z 2021-12-21T21:59:09.263231Z 2021-12-21T21:59:09.205197Z 2021-12-21T21:59:09.127775Z 2021-12-21T21:59:09.020033Z 2021-12-21T21:59:09.086820Z 2021-12-21T21:59:08.886225Z 2021-12-21T21:59:08.724477Z 2021-12-21T21:59:08.655679Z 2021-12-21T21:59:08.617935Z 2021-12-21T21:59:08.579549Z 2021-12-21T21:59:08.543551Z 2021-12-21T21:59:08.488142Z 2021-12-21T21:59:08.348164Z 2021-12-21T21:59:08.225254Z 2021-12-21T21:59:08.075256Z 2021-12-21T21:59:07.950159Z 2021-12-21T21:59:07.844169Z 2021-12-21T21:59:07.758036Z 2021-12-21T21:59:07.676194Z 2021-12-21T21:59:07.579090Z 2021-12-21T21:59:07.480032Z 2021-12-21T21:59:07.375336Z 2021-12-21T21:59:07.286184Z 2021-12-21T21:59:07.247223Z 2021-12-21T21:59:07.195045Z 2021-12-21T21:59:07.146068Z 2021-12-21T21:59:07.108318Z 2021-12-21T21:59:07.068376Z 2021-12-21T21:59:07.015086Z 2021-12-21T21:59:06.956035Z 2021-12-21T21:59:06.902084Z 2021-12-21T21:59:06.840269Z 2021-12-21T21:59:06.720132Z 2021-12-21T21:59:06.613194Z 2021-12-21T21:59:06.438325Z 2021-12-21T21:59:06.365692Z 2021-12-21T21:59:06.310588Z 2021-12-21T21:59:06.273434Z 2021-12-21T21:59:06.201173Z 2021-12-21T21:59:06.136806Z 2021-12-21T21:59:06.088103Z 2021-12-21T21:59:06.022870Z 2021-12-21T21:59:05.986887Z 2021-12-21T21:59:05.949999Z 2021-12-21T21:59:05.903827Z 2021-12-21T21:59:05.866056Z 2021-12-21T21:59:05.817248Z 2021-12-21T21:59:05.738896Z 2021-12-21T21:59:05.778340Z 2021-12-21T21:59:05.702349Z 2021-12-21T21:59:05.655829Z 2021-12-21T21:59:05.618096Z 2021-12-21T21:59:05.569307Z 2021-12-21T21:59:05.522157Z 2021-12-21T21:59:05.485203Z 2021-12-21T21:59:05.432492Z 2021-12-21T21:59:05.388408Z 2021-12-21T21:59:05.346615Z 2021-12-21T21:59:05.309116Z 2021-12-21T21:59:05.275404Z 2021-12-21T21:59:05.234264Z 2021-12-21T21:59:05.180077Z 2021-12-21T21:59:05.124333Z 2021-12-21T21:59:05.071047Z 2021-12-21T21:59:04.989289Z 2021-12-21T21:59:04.874297Z 2021-12-21T21:59:04.827314Z 2021-12-21T21:59:04.764300Z 2021-12-21T21:59:04.682119Z 2021-12-21T21:59:04.583361Z 2021-12-21T21:59:04.455684Z 2021-12-21T21:58:56.263755Z 2021-12-21T21:58:56.053306Z 2021-12-21T21:58:55.719049Z 2021-12-21T21:58:55.466268Z 2021-12-21T21:58:55.166193Z 2021-12-21T21:58:54.863888Z 2021-12-21T21:58:54.674828Z 2021-12-21T21:58:54.420345Z 2021-12-21T21:58:54.223433Z 2021-12-21T21:58:54.015556Z 2021-12-21T21:58:53.745142Z 2021-12-21T21:58:53.463876Z 2021-12-21T21:58:53.209841Z 2021-12-21T21:58:50.603323Z 2021-12-21T21:58:50.517565Z 2021-12-21T21:58:50.461189Z 2021-12-21T21:58:50.402943Z 2021-12-21T22:03:29.068010Z 2021-12-21T22:03:29.007876Z 2021-12-21T22:03:28.954036Z 2021-12-21T22:03:28.878930Z 2021-12-21T22:03:28.828325Z 2021-12-21T22:03:28.713900Z 2021-12-21T22:03:28.605048Z 2021-12-21T22:03:28.487870Z 2021-12-21T22:03:28.396019Z 2021-12-21T22:03:28.347910Z 2021-12-21T22:03:28.297111Z 2021-12-21T22:03:28.234809Z 2021-12-21T22:03:28.182835Z 2021-12-21T22:03:28.136979Z 2021-12-21T22:03:28.078939Z 2021-12-21T22:03:28.025952Z 2021-12-21T22:03:27.979061Z 2021-12-21T22:03:27.903947Z 2021-12-21T22:03:27.819080Z 2021-12-21T22:03:27.761759Z 2021-12-21T22:03:27.693121Z 2021-12-21T22:03:27.640043Z 2021-12-21T22:03:27.566799Z 2021-12-21T22:03:27.497079Z 2021-12-21T22:03:27.429118Z 2021-12-21T22:03:27.361076Z 2021-12-21T22:03:27.281098Z 2021-12-21T22:03:27.222148Z 2021-12-21T22:03:27.149916Z 2021-12-21T22:03:27.088973Z 2021-12-21T22:03:27.007943Z 2021-12-21T22:03:26.937027Z 2021-12-21T22:03:26.874516Z 2021-12-21T22:03:26.790915Z 2021-12-21T22:03:26.677542Z 2021-12-21T22:03:26.543006Z 2021-12-21T22:03:26.416141Z 2021-12-21T22:03:26.306067Z 2021-12-21T22:03:26.198969Z 2021-12-21T22:03:26.038956Z 2021-12-21T22:03:25.883963Z 2021-12-21T22:03:25.768829Z 2021-12-21T22:03:25.688795Z 2021-12-21T22:03:25.591028Z 2021-12-21T22:03:25.505086Z 2021-12-21T22:03:25.387689Z 2021-12-21T22:03:25.236993Z 2021-12-21T22:03:11.219588Z 2021-12-21T22:03:11.176817Z 2021-12-21T22:03:11.133669Z 2021-12-21T22:03:11.091557Z 2021-12-21T22:03:11.049904Z 2021-12-21T22:03:11.004598Z 2021-12-21T22:03:10.955625Z 2021-12-21T22:03:10.894362Z 2021-12-21T22:03:10.812375Z 2021-12-21T22:03:10.717943Z 2021-12-21T22:03:10.645782Z 2021-12-21T22:03:10.602334Z 2021-12-21T22:03:10.565121Z 2021-12-21T22:03:10.528671Z 2021-12-21T22:03:10.488153Z 2021-12-21T22:03:10.420022Z 2021-12-21T22:03:10.360074Z 2021-12-21T22:03:10.289385Z 2021-12-21T22:03:10.212234Z 2021-12-21T22:03:10.147639Z 2021-12-21T22:03:09.754154Z 2021-12-21T22:03:10.043896Z 2021-12-21T22:03:09.887926Z 2021-12-21T22:03:09.510191Z 2021-12-21T22:03:09.207984Z 2021-12-21T22:03:09.147281Z 2021-12-21T22:03:09.029542Z 2021-12-21T22:03:08.748010Z 2021-12-21T22:03:08.420381Z 2021-12-21T22:03:08.145196Z 2021-12-21T22:03:07.782097Z 2021-12-21T22:03:07.653017Z 2021-12-21T22:03:07.533520Z 2021-12-21T22:03:07.437431Z 2021-12-21T22:03:07.145035Z 2021-12-21T22:03:06.968382Z 2021-12-21T22:03:06.807803Z 2021-12-21T22:03:06.676877Z 2021-12-21T22:03:06.459642Z 2021-12-21T22:03:06.107256Z 2021-12-21T22:02:59.075276Z 2021-12-21T22:02:59.031952Z 2021-12-21T22:02:58.991377Z 2021-12-21T22:02:58.940542Z 2021-12-21T22:02:58.875828Z 2021-12-21T22:02:58.782795Z 2021-12-21T22:02:58.699107Z 2021-12-21T22:02:58.561720Z 2021-12-21T22:02:57.656468Z 2021-12-21T22:02:58.516085Z 2021-12-21T22:02:57.606768Z 2021-12-21T22:02:57.567808Z 2021-12-21T22:02:57.534333Z", + "modified_diff": 43.5601044, + "started": "2021-12-21T21:46:48.978152Z 2021-12-21T21:46:48.270634Z 2021-12-21T21:46:48.032182Z 2021-12-21T21:46:47.823202Z 2021-12-21T21:46:47.710698Z 2021-12-21T21:46:47.128415Z 2021-12-21T21:46:47.056063Z 2021-12-21T21:46:46.478403Z 2021-12-21T21:46:46.197339Z 2021-12-21T21:46:45.991469Z 2021-12-21T21:46:45.797275Z 2021-12-21T21:46:45.352411Z 2021-12-21T21:46:45.162244Z 2021-12-21T21:46:31.199393Z 2021-12-21T21:46:31.192723Z 2021-12-21T21:46:29.801478Z 2021-12-21T21:46:29.530310Z 2021-12-21T21:46:28.460809Z 2021-12-21T21:46:27.964900Z 2021-12-21T21:46:27.203039Z 2021-12-21T21:46:27.372088Z 2021-12-21T21:46:26.663598Z 2021-12-21T21:46:27.136427Z 2021-12-21T21:46:26.182822Z 2021-12-21T21:46:25.934139Z 2021-12-21T21:46:21.527202Z 2021-12-21T21:46:21.489424Z 2021-12-21T21:46:21.265369Z 2021-12-21T21:46:21.116588Z 2021-12-21T21:46:20.928552Z 2021-12-21T21:46:20.909240Z 2021-12-21T21:46:19.892890Z 2021-12-21T21:46:19.825194Z 2021-12-21T21:46:19.707568Z 2021-12-21T21:46:19.287030Z 2021-12-21T21:47:17.224723Z 2021-12-21T21:47:16.472555Z 2021-12-21T21:47:16.234276Z 2021-12-21T21:47:15.835741Z 2021-12-21T21:47:15.745981Z 2021-12-21T21:47:15.260999Z 2021-12-21T21:47:15.115581Z 2021-12-21T21:47:14.835771Z 2021-12-21T21:47:15.474584Z 2021-12-21T21:47:14.473566Z 2021-12-21T21:47:14.493157Z 2021-12-21T21:47:14.205297Z 2021-12-21T21:47:13.643507Z 2021-12-21T21:47:13.617502Z 2021-12-21T21:47:13.176132Z 2021-12-21T21:47:05.838770Z 2021-12-21T21:47:05.600915Z 2021-12-21T21:47:05.297610Z 2021-12-21T21:47:04.754792Z 2021-12-21T21:47:04.543374Z 2021-12-21T21:47:04.245904Z 2021-12-21T21:47:03.949582Z 2021-12-21T21:47:03.423157Z 2021-12-21T21:47:03.143077Z 2021-12-21T21:47:02.393949Z 2021-12-21T21:47:12.990901Z 2021-12-21T21:47:02.004512Z 2021-12-21T21:47:01.873773Z 2021-12-21T21:47:01.317869Z 2021-12-21T21:47:00.970853Z 2021-12-21T21:47:12.834758Z 2021-12-21T21:47:00.757145Z 2021-12-21T21:47:00.402963Z 2021-12-21T21:47:00.201817Z 2021-12-21T21:46:59.609725Z 2021-12-21T21:46:59.199555Z 2021-12-21T21:46:59.093830Z 2021-12-21T21:46:59.079459Z 2021-12-21T21:46:58.609060Z 2021-12-21T21:46:58.436235Z 2021-12-21T21:46:57.669777Z 2021-12-21T21:46:58.139551Z 2021-12-21T21:46:57.394644Z 2021-12-21T21:46:57.000287Z 2021-12-21T21:46:56.679967Z 2021-12-21T21:46:56.219854Z 2021-12-21T21:46:55.970755Z 2021-12-21T21:46:55.485417Z 2021-12-21T21:46:55.233251Z 2021-12-21T21:46:54.744021Z 2021-12-21T21:46:54.213685Z 2021-12-21T21:46:54.091627Z 2021-12-21T21:46:53.745556Z 2021-12-21T21:46:53.205381Z 2021-12-21T21:46:52.889999Z 2021-12-21T21:46:52.548661Z 2021-12-21T21:46:52.277351Z 2021-12-21T21:46:51.747366Z 2021-12-21T21:46:51.517835Z 2021-12-21T21:46:51.238590Z 2021-12-21T21:46:50.803972Z 2021-12-21T21:46:50.103347Z 2021-12-21T21:46:49.510131Z 2021-12-21T21:46:49.492744Z 2021-12-21T21:46:49.153664Z 2021-12-21T21:51:46.600752Z 2021-12-21T21:51:46.548013Z 2021-12-21T21:51:45.916798Z 2021-12-21T21:51:45.619258Z 2021-12-21T21:51:45.276157Z 2021-12-21T21:51:44.967736Z 2021-12-21T21:51:44.767826Z 2021-12-21T21:51:44.480115Z 2021-12-21T21:51:44.077923Z 2021-12-21T21:51:43.825678Z 2021-12-21T21:51:43.398835Z 2021-12-21T21:51:43.251740Z 2021-12-21T21:51:42.728033Z 2021-12-21T21:51:42.327246Z 2021-12-21T21:51:41.891501Z 2021-12-21T21:51:41.440330Z 2021-12-21T21:51:40.944837Z 2021-12-21T21:51:40.810440Z 2021-12-21T21:51:40.349907Z 2021-12-21T21:51:39.806249Z 2021-12-21T21:51:39.284558Z 2021-12-21T21:51:39.136014Z 2021-12-21T21:51:38.403742Z 2021-12-21T21:51:38.321790Z 2021-12-21T21:51:37.774351Z 2021-12-21T21:51:37.446348Z 2021-12-21T21:51:36.733631Z 2021-12-21T21:51:36.625736Z 2021-12-21T21:51:36.096598Z 2021-12-21T21:51:36.133300Z 2021-12-21T21:51:35.476297Z 2021-12-21T21:51:35.020408Z 2021-12-21T21:51:34.173584Z 2021-12-21T21:51:33.934639Z 2021-12-21T21:51:33.683636Z 2021-12-21T21:51:33.218174Z 2021-12-21T21:51:32.835238Z 2021-12-21T21:51:32.696000Z 2021-12-21T21:51:31.657500Z 2021-12-21T21:51:30.783834Z 2021-12-21T21:51:31.346704Z 2021-12-21T21:51:30.309079Z 2021-12-21T21:51:30.078962Z 2021-12-21T21:51:30.248784Z 2021-12-21T21:51:29.486314Z 2021-12-21T21:51:29.141758Z 2021-12-21T21:51:28.724565Z 2021-12-21T21:51:28.391917Z 2021-12-21T21:51:27.905637Z 2021-12-21T21:51:27.683678Z 2021-12-21T21:51:27.044914Z 2021-12-21T21:51:26.729362Z 2021-12-21T21:51:26.429814Z 2021-12-21T21:51:25.930918Z 2021-12-21T21:51:25.899950Z 2021-12-21T21:51:25.431706Z 2021-12-21T21:51:25.373273Z 2021-12-21T21:51:25.068131Z 2021-12-21T21:51:24.634861Z 2021-12-21T21:51:24.351351Z 2021-12-21T21:51:24.345898Z 2021-12-21T21:51:23.818439Z 2021-12-21T21:51:23.560857Z 2021-12-21T21:51:23.179732Z 2021-12-21T21:51:22.970603Z 2021-12-21T21:51:22.628099Z 2021-12-21T21:51:22.552751Z 2021-12-21T21:51:22.262915Z 2021-12-21T21:51:22.033873Z 2021-12-21T21:51:21.836556Z 2021-12-21T21:51:12.785686Z 2021-12-21T21:51:12.475738Z 2021-12-21T21:51:12.019759Z 2021-12-21T21:51:11.743736Z 2021-12-21T21:51:11.231216Z 2021-12-21T21:51:11.183482Z 2021-12-21T21:51:10.779551Z 2021-12-21T21:51:10.510170Z 2021-12-21T21:51:09.884516Z 2021-12-21T21:51:09.599895Z 2021-12-21T21:51:08.968667Z 2021-12-21T21:51:08.761470Z 2021-12-21T21:51:08.259112Z 2021-12-21T21:51:08.230508Z 2021-12-21T21:51:07.809921Z 2021-12-21T21:51:08.359262Z 2021-12-21T21:51:07.512438Z 2021-12-21T21:51:06.320555Z 2021-12-21T21:51:06.198103Z 2021-12-21T21:50:56.670259Z 2021-12-21T21:50:56.286569Z 2021-12-21T21:50:56.730112Z 2021-12-21T21:50:53.488485Z 2021-12-21T21:50:53.675415Z 2021-12-21T21:50:53.281806Z 2021-12-21T21:50:53.200581Z 2021-12-21T21:50:53.010253Z 2021-12-21T21:50:52.534192Z 2021-12-21T21:50:52.458423Z 2021-12-21T21:50:52.358353Z 2021-12-21T21:55:42.350792Z 2021-12-21T21:55:41.931996Z 2021-12-21T21:55:41.762643Z 2021-12-21T21:55:41.277907Z 2021-12-21T21:55:41.132385Z 2021-12-21T21:55:40.715620Z 2021-12-21T21:55:40.496447Z 2021-12-21T21:55:40.074949Z 2021-12-21T21:55:39.855198Z 2021-12-21T21:55:31.699758Z 2021-12-21T21:55:31.507039Z 2021-12-21T21:55:31.182442Z 2021-12-21T21:55:30.924114Z 2021-12-21T21:55:30.275307Z 2021-12-21T21:55:30.195646Z 2021-12-21T21:55:29.209640Z 2021-12-21T21:55:29.140718Z 2021-12-21T21:55:28.309867Z 2021-12-21T21:55:27.993797Z 2021-12-21T21:55:27.770474Z 2021-12-21T21:55:39.624077Z 2021-12-21T21:55:27.404048Z 2021-12-21T21:55:27.090382Z 2021-12-21T21:55:26.723028Z 2021-12-21T21:55:26.265940Z 2021-12-21T21:55:25.735835Z 2021-12-21T21:55:25.446219Z 2021-12-21T21:55:25.288496Z 2021-12-21T21:55:24.803515Z 2021-12-21T21:55:24.546003Z 2021-12-21T21:55:24.298007Z 2021-12-21T21:55:23.727447Z 2021-12-21T21:55:23.464220Z 2021-12-21T21:55:23.369034Z 2021-12-21T21:55:22.941909Z 2021-12-21T21:55:22.679488Z 2021-12-21T21:55:22.014782Z 2021-12-21T21:55:21.919390Z 2021-12-21T21:55:21.362845Z 2021-12-21T21:55:21.091110Z 2021-12-21T21:55:20.908846Z 2021-12-21T21:55:20.351440Z 2021-12-21T21:55:20.510113Z 2021-12-21T21:55:19.923059Z 2021-12-21T21:55:19.379715Z 2021-12-21T21:55:19.080122Z 2021-12-21T21:55:18.574759Z 2021-12-21T21:55:18.287914Z 2021-12-21T21:55:17.719894Z 2021-12-21T21:55:17.212893Z 2021-12-21T21:55:16.785932Z 2021-12-21T21:55:16.413451Z 2021-12-21T21:55:16.068383Z 2021-12-21T21:55:15.761754Z 2021-12-21T21:55:15.513889Z 2021-12-21T21:55:14.961310Z 2021-12-21T21:55:14.236818Z 2021-12-21T21:55:14.031175Z 2021-12-21T21:55:13.750954Z 2021-12-21T21:55:13.431116Z 2021-12-21T21:55:12.632779Z 2021-12-21T21:55:12.419264Z 2021-12-21T21:55:11.685752Z 2021-12-21T21:55:11.320785Z 2021-12-21T21:55:10.787224Z 2021-12-21T21:55:10.636578Z 2021-12-21T21:55:09.638415Z 2021-12-21T21:55:09.665348Z 2021-12-21T21:55:09.103815Z 2021-12-21T21:55:08.711623Z 2021-12-21T21:55:08.315029Z 2021-12-21T21:55:08.041441Z 2021-12-21T21:55:07.942750Z 2021-12-21T21:55:07.551243Z 2021-12-21T21:55:07.327250Z 2021-12-21T21:55:07.291108Z 2021-12-21T21:55:06.586619Z 2021-12-21T21:55:06.583073Z 2021-12-21T21:55:06.267156Z 2021-12-21T21:55:06.080308Z 2021-12-21T21:54:57.451990Z 2021-12-21T21:54:57.130806Z 2021-12-21T21:54:56.877091Z 2021-12-21T21:54:55.557139Z 2021-12-21T21:54:54.863865Z 2021-12-21T21:54:54.586730Z 2021-12-21T21:54:54.183856Z 2021-12-21T21:54:53.199809Z 2021-12-21T21:54:52.613928Z 2021-12-21T21:54:52.915692Z 2021-12-21T21:54:51.906899Z 2021-12-21T21:54:51.525344Z 2021-12-21T21:54:51.366495Z 2021-12-21T21:54:51.523669Z 2021-12-21T21:54:50.401891Z 2021-12-21T21:54:46.937356Z 2021-12-21T21:54:46.811727Z 2021-12-21T21:54:46.563558Z 2021-12-21T21:54:45.670498Z 2021-12-21T21:54:45.011408Z 2021-12-21T21:59:40.455779Z 2021-12-21T21:59:40.364785Z 2021-12-21T21:59:36.624916Z 2021-12-21T21:59:36.467335Z 2021-12-21T21:59:35.952897Z 2021-12-21T21:59:35.654098Z 2021-12-21T21:59:35.261556Z 2021-12-21T21:59:34.751930Z 2021-12-21T21:59:34.301008Z 2021-12-21T21:59:34.026233Z 2021-12-21T21:59:33.575039Z 2021-12-21T21:59:33.136106Z 2021-12-21T21:59:32.744742Z 2021-12-21T21:59:32.385265Z 2021-12-21T21:59:32.378606Z 2021-12-21T21:59:31.642655Z 2021-12-21T21:59:31.728873Z 2021-12-21T21:59:31.344683Z 2021-12-21T21:59:31.344860Z 2021-12-21T21:59:31.004858Z 2021-12-21T21:59:30.554882Z 2021-12-21T21:59:30.168987Z 2021-12-21T21:59:29.761852Z 2021-12-21T21:59:30.007789Z 2021-12-21T21:59:29.272690Z 2021-12-21T21:59:29.187881Z 2021-12-21T21:59:28.849862Z 2021-12-21T21:59:28.605196Z 2021-12-21T21:59:28.286962Z 2021-12-21T21:59:28.173916Z 2021-12-21T21:59:27.633874Z 2021-12-21T21:59:27.301705Z 2021-12-21T21:59:27.140649Z 2021-12-21T21:59:26.682931Z 2021-12-21T21:59:26.431886Z 2021-12-21T21:59:26.096029Z 2021-12-21T21:59:25.772881Z 2021-12-21T21:59:25.403021Z 2021-12-21T21:59:25.272738Z 2021-12-21T21:59:24.905796Z 2021-12-21T21:59:24.485800Z 2021-12-21T21:59:24.155662Z 2021-12-21T21:59:23.742878Z 2021-12-21T21:59:23.270190Z 2021-12-21T21:59:22.697869Z 2021-12-21T21:59:22.449135Z 2021-12-21T21:59:22.291148Z 2021-12-21T21:59:21.893405Z 2021-12-21T21:59:21.701782Z 2021-12-21T21:59:21.273199Z 2021-12-21T21:59:20.783674Z 2021-12-21T21:59:20.527241Z 2021-12-21T21:59:19.970274Z 2021-12-21T21:59:19.442320Z 2021-12-21T21:59:19.153597Z 2021-12-21T21:59:18.977079Z 2021-12-21T21:59:18.556751Z 2021-12-21T21:59:18.120760Z 2021-12-21T21:59:18.034536Z 2021-12-21T21:59:17.306858Z 2021-12-21T21:59:17.729135Z 2021-12-21T21:59:17.142601Z 2021-12-21T21:59:16.543779Z 2021-12-21T21:59:16.704932Z 2021-12-21T21:59:15.791574Z 2021-12-21T21:59:15.560414Z 2021-12-21T21:59:14.861055Z 2021-12-21T21:59:14.569697Z 2021-12-21T21:59:13.926553Z 2021-12-21T21:59:13.703048Z 2021-12-21T21:59:13.355527Z 2021-12-21T21:59:13.509748Z 2021-12-21T21:59:12.865300Z 2021-12-21T21:59:12.776208Z 2021-12-21T21:59:12.294427Z 2021-12-21T21:59:12.091938Z 2021-12-21T21:59:11.723157Z 2021-12-21T21:59:11.445255Z 2021-12-21T21:59:11.186460Z 2021-12-21T21:59:11.066410Z 2021-12-21T21:59:10.901004Z 2021-12-21T21:59:10.872821Z 2021-12-21T21:59:10.610177Z 2021-12-21T21:59:01.996565Z 2021-12-21T21:59:01.482645Z 2021-12-21T21:59:01.339611Z 2021-12-21T21:59:00.992904Z 2021-12-21T21:59:00.842052Z 2021-12-21T21:59:00.382450Z 2021-12-21T21:59:00.053485Z 2021-12-21T21:58:59.704457Z 2021-12-21T21:58:59.212304Z 2021-12-21T21:58:59.028616Z 2021-12-21T21:58:58.224956Z 2021-12-21T21:58:57.788649Z 2021-12-21T21:58:57.699700Z 2021-12-21T21:58:51.873730Z 2021-12-21T21:58:51.450774Z 2021-12-21T21:58:51.059495Z 2021-12-21T21:58:50.953605Z 2021-12-21T22:03:44.627438Z 2021-12-21T22:03:44.780684Z 2021-12-21T22:03:44.016628Z 2021-12-21T22:03:43.961743Z 2021-12-21T22:03:43.296104Z 2021-12-21T22:03:43.178519Z 2021-12-21T22:03:43.061955Z 2021-12-21T22:03:42.511622Z 2021-12-21T22:03:42.249462Z 2021-12-21T22:03:41.674825Z 2021-12-21T22:03:41.361020Z 2021-12-21T22:03:41.106736Z 2021-12-21T22:03:40.857146Z 2021-12-21T22:03:40.502172Z 2021-12-21T22:03:40.079090Z 2021-12-21T22:03:39.793008Z 2021-12-21T22:03:39.089642Z 2021-12-21T22:03:38.726291Z 2021-12-21T22:03:38.508408Z 2021-12-21T22:03:38.402658Z 2021-12-21T22:03:38.015399Z 2021-12-21T22:03:37.564367Z 2021-12-21T22:03:37.215111Z 2021-12-21T22:03:37.105868Z 2021-12-21T22:03:36.317042Z 2021-12-21T22:03:36.115820Z 2021-12-21T22:03:35.696916Z 2021-12-21T22:03:35.194299Z 2021-12-21T22:03:34.970091Z 2021-12-21T22:03:34.553166Z 2021-12-21T22:03:34.175186Z 2021-12-21T22:03:33.689602Z 2021-12-21T22:03:33.176864Z 2021-12-21T22:03:32.897009Z 2021-12-21T22:03:32.821120Z 2021-12-21T22:03:32.382905Z 2021-12-21T22:03:32.326196Z 2021-12-21T22:03:31.754413Z 2021-12-21T22:03:31.563863Z 2021-12-21T22:03:31.336127Z 2021-12-21T22:03:30.976169Z 2021-12-21T22:03:30.791429Z 2021-12-21T22:03:30.292661Z 2021-12-21T22:03:30.108055Z 2021-12-21T22:03:29.756026Z 2021-12-21T22:03:29.678764Z 2021-12-21T22:03:29.529443Z 2021-12-21T22:03:23.038665Z 2021-12-21T22:03:22.755083Z 2021-12-21T22:03:22.437418Z 2021-12-21T22:03:22.127297Z 2021-12-21T22:03:21.931286Z 2021-12-21T22:03:21.605417Z 2021-12-21T22:03:20.698250Z 2021-12-21T22:03:20.689387Z 2021-12-21T22:03:20.187374Z 2021-12-21T22:03:20.108678Z 2021-12-21T22:03:19.989172Z 2021-12-21T22:03:19.207933Z 2021-12-21T22:03:19.193669Z 2021-12-21T22:03:18.518015Z 2021-12-21T22:03:18.452283Z 2021-12-21T22:03:17.943873Z 2021-12-21T22:03:17.970617Z 2021-12-21T22:03:17.836019Z 2021-12-21T22:03:17.130888Z 2021-12-21T22:03:16.990204Z 2021-12-21T22:03:15.786302Z 2021-12-21T22:03:16.543130Z 2021-12-21T22:03:16.178120Z 2021-12-21T22:03:15.569607Z 2021-12-21T22:03:15.345728Z 2021-12-21T22:03:15.285139Z 2021-12-21T22:03:15.153051Z 2021-12-21T22:03:14.621019Z 2021-12-21T22:03:14.577771Z 2021-12-21T22:03:13.913212Z 2021-12-21T22:03:13.837461Z 2021-12-21T22:03:13.468316Z 2021-12-21T22:03:13.297031Z 2021-12-21T22:03:12.996233Z 2021-12-21T22:03:12.591806Z 2021-12-21T22:03:12.357954Z 2021-12-21T22:03:12.082546Z 2021-12-21T22:03:11.970017Z 2021-12-21T22:03:11.776577Z 2021-12-21T22:03:11.593182Z 2021-12-21T22:03:03.357014Z 2021-12-21T22:03:03.260825Z 2021-12-21T22:03:02.335474Z 2021-12-21T22:03:01.523835Z 2021-12-21T22:03:01.674140Z 2021-12-21T22:03:00.770362Z 2021-12-21T22:03:00.456190Z 2021-12-21T22:02:59.794752Z 2021-12-21T22:02:58.291787Z 2021-12-21T22:02:59.609287Z 2021-12-21T22:02:58.274896Z 2021-12-21T22:02:58.068581Z 2021-12-21T22:02:57.993141Z", + "started_diff": 53.161838599999996, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 93, 92, 90, 89, 88, 86, 85, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 72, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 154, 153, 152, 150, 149, 148, 147, 145, 144, 143, 142, 141, 139, 138, 137, 136, 135, 134, 132, 131, 130, 129, 128, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 171, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 271, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 371, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, 471]" + }, + "started": "2021-12-21T22:29:06.167643+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-22T14_32_45_409_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-22T14_32_45_409_0000-chatty.json new file mode 100644 index 00000000..ff49e08d --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-22T14_32_45_409_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-22T14_32_45_409_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-22T16:20:38.186726+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-23T03_04_04_157_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-23T03_04_04_157_0000-chatty.json new file mode 100644 index 00000000..90db85f4 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-23T03_04_04_157_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-23T03_04_04_157_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-23T04:57:13.041104+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-24T03_04_04_13_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-24T03_04_04_13_0000-chatty.json new file mode 100644 index 00000000..857855a5 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-24T03_04_04_13_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-24T03_04_04_13_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-24T04:48:52.557781+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-25T03_04_03_800_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-25T03_04_03_800_0000-chatty.json new file mode 100644 index 00000000..bbd68efd --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-25T03_04_03_800_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-25T03_04_03_800_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-25T04:50:41.667110+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-26T03_04_04_403_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-26T03_04_04_403_0000-chatty.json new file mode 100644 index 00000000..64dd368f --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-26T03_04_04_403_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-26T03_04_04_403_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-26T04:50:33.658457+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-27T03_04_03_816_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-27T03_04_03_816_0000-chatty.json new file mode 100644 index 00000000..e26d7107 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-27T03_04_03_816_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-27T03_04_03_816_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-27T04:52:36.268867+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-28T03_04_04_393_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-28T03_04_04_393_0000-chatty.json new file mode 100644 index 00000000..b5a686b4 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-28T03_04_04_393_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-28T03_04_04_393_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-28T04:46:34.270256+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2021-12-29T03_04_04_157_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2021-12-29T03_04_04_157_0000-chatty.json new file mode 100644 index 00000000..e92afda4 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2021-12-29T03_04_04_157_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2021-12-29T03_04_04_157_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2021-12-29T04:47:12.879546+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-07T03_04_04_285_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-07T03_04_04_285_0000-chatty.json new file mode 100644 index 00000000..e17e0299 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-07T03_04_04_285_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-07T03_04_04_285_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-07T04:58:45.064434+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-08T03_04_03_581_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-08T03_04_03_581_0000-chatty.json new file mode 100644 index 00000000..4568a663 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-08T03_04_03_581_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-08T03_04_03_581_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-08T04:45:51.148621+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-09T03_04_04_81_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-09T03_04_04_81_0000-chatty.json new file mode 100644 index 00000000..82495a8e --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-09T03_04_04_81_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-09T03_04_04_81_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-09T04:46:02.689437+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-10T03_04_04_437_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-10T03_04_04_437_0000-chatty.json new file mode 100644 index 00000000..2e2b3bfe --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-10T03_04_04_437_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-10T03_04_04_437_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-10T04:54:10.540536+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-api.json new file mode 100644 index 00000000..e1e00f43 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-api.json @@ -0,0 +1,1244 @@ +{ + "ended": "2022-01-10T17:54:03.140829+00:00", + "id": "run-2022-01-10T14_44_55_783_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2343.5999999999995, + "max": 25138.2, + "mean": 5261.694316950988, + "median": 3289.5, + "min": 1384.4666666666667, + "non_zero_mean": 5261.694316950988, + "non_zero_median": 3289.5, + "percentile25": 2821.8166666666666, + "percentile75": 5165.416666666666, + "percentile90": 11328.5, + "percentile99": 23543.196666666685, + "percentile999": 24877.634333333346, + "range": 23753.733333333334, + "samples": 266, + "stdev": 4668.124094001253, + "sum": 1399610.6883089624 + } + }, + "cpu": { + "system": { + "iqr": 0.1888333333333274, + "max": 1.763333333333329, + "mean": 0.2683483916522641, + "median": 0.11033333333333342, + "min": 0.029333333333333184, + "non_zero_mean": 0.2683483916522641, + "non_zero_median": 0.11033333333333342, + "percentile25": 0.06083333333333579, + "percentile75": 0.2496666666666632, + "percentile90": 0.831333333333337, + "percentile99": 1.679000000000005, + "percentile999": 1.7578566666666624, + "range": 1.7339999999999958, + "samples": 266, + "stdev": 0.3820278323830876, + "sum": 71.38067217950224 + }, + "user": { + "iqr": 0.5736666666666755, + "max": 6.416666666666697, + "mean": 1.3491279151477589, + "median": 0.7600000000000137, + "min": 0.3186666666667255, + "non_zero_mean": 1.3491279151477589, + "non_zero_median": 0.7600000000000137, + "percentile25": 0.5833333333333409, + "percentile75": 1.1570000000000165, + "percentile90": 3.5189999999999904, + "percentile99": 6.208133333333317, + "percentile999": 6.4004133333333595, + "range": 6.097999999999971, + "samples": 266, + "stdev": 1.4356795592778968, + "sum": 358.86802542930377 + } + }, + "entropy_available_bits": { + "iqr": 2.19997777481442, + "max": 213.86666666666667, + "mean": 19.6987470344528, + "median": 1.7333333333333334, + "min": 0.2, + "non_zero_mean": 19.6987470344528, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.6666888918522469, + "percentile75": 2.8666666666666667, + "percentile90": 6.133333333333333, + "percentile99": 211.62666666666667, + "percentile999": 213.69000000000003, + "range": 213.66666666666669, + "samples": 266, + "stdev": 55.32859539708843, + "sum": 5239.866711164443 + }, + "memory": { + "used": { + "iqr": 377971712.0, + "max": 16877101056.0, + "mean": 6426867250.045113, + "median": 5891751936.0, + "min": 5537144832.0, + "non_zero_mean": 6426867250.045113, + "non_zero_median": 5891751936.0, + "percentile25": 5758755840.0, + "percentile75": 6136727552.0, + "percentile90": 6817050624.0, + "percentile99": 15694512742.400011, + "percentile999": 16807397355.520004, + "range": 11339956224.0, + "samples": 266, + "stdev": 1880160512.637399, + "sum": 1709546688512.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 26.6, + "mean": 0.14210526315789473, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 5.4, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.7800000000000031, + "percentile999": 20.628666666666973, + "range": 26.6, + "samples": 266, + "stdev": 1.6597562132464965, + "sum": 37.80000000000001 + } + }, + "writes_completed": { + "total": { + "iqr": 86.66666666666666, + "max": 1588.7333333333333, + "mean": 117.92205774495129, + "median": 26.79967557718374, + "min": 0.0, + "non_zero_mean": 119.26717627436138, + "non_zero_median": 27.266666666666666, + "percentile25": 1.4, + "percentile75": 88.06666666666666, + "percentile90": 312.0333333333333, + "percentile99": 1296.1833333333336, + "percentile999": 1542.870666666669, + "range": 1588.7333333333333, + "samples": 266, + "stdev": 256.9423570953857, + "sum": 31367.267360157046 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 981.7833333333333, + "max": 3456.0, + "mean": 1616.39649122807, + "median": 1469.0333333333333, + "min": 505.93333333333334, + "non_zero_mean": 1616.39649122807, + "non_zero_median": 1469.0333333333333, + "percentile25": 1156.3666666666668, + "percentile75": 2138.15, + "percentile90": 2420.4, + "percentile99": 3085.0666666666675, + "percentile999": 3389.29066666667, + "range": 2950.0666666666666, + "samples": 266, + "stdev": 606.1245568662265, + "sum": 429961.4666666666 + } + }, + "cpu": { + "system": { + "iqr": 0.04266666666666504, + "max": 0.24599999999999983, + "mean": 0.08329323308270677, + "median": 0.07133333333333192, + "min": 0.016000000000000604, + "non_zero_mean": 0.08329323308270677, + "non_zero_median": 0.07133333333333192, + "percentile25": 0.054000000000000624, + "percentile75": 0.09666666666666567, + "percentile90": 0.15233333333333404, + "percentile99": 0.23103333333333323, + "percentile999": 0.24582333333333384, + "range": 0.22999999999999923, + "samples": 266, + "stdev": 0.04477418838628225, + "sum": 22.156000000000002 + }, + "user": { + "iqr": 0.12166666666666592, + "max": 0.8960000000000036, + "mean": 0.1982155388471178, + "median": 0.18833333333333446, + "min": 0.018000000000002576, + "non_zero_mean": 0.1982155388471178, + "non_zero_median": 0.18833333333333446, + "percentile25": 0.13083333333333513, + "percentile75": 0.25250000000000106, + "percentile90": 0.33299999999999746, + "percentile99": 0.4905000000000028, + "percentile999": 0.8097866666666728, + "range": 0.878000000000001, + "samples": 266, + "stdev": 0.10540093520116812, + "sum": 52.72533333333333 + } + }, + "entropy_available_bits": { + "iqr": 1.7833333333333334, + "max": 212.2, + "mean": 11.148120300751879, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 22.465151515151515, + "non_zero_median": 1.8333333333333335, + "percentile25": 0.0, + "percentile75": 1.7833333333333334, + "percentile90": 3.3, + "percentile99": 211.78, + "percentile999": 212.12933333333334, + "range": 212.2, + "samples": 266, + "stdev": 45.471010307931884, + "sum": 2965.4000000000005 + }, + "memory": { + "used": { + "iqr": 22065152.0, + "max": 1074667520.0, + "mean": 434313323.7894737, + "median": 392278016.0, + "min": 371257344.0, + "non_zero_mean": 434313323.7894737, + "non_zero_median": 392278016.0, + "percentile25": 385346560.0, + "percentile75": 407411712.0, + "percentile90": 520632320.0, + "percentile99": 1022259200.0, + "percentile999": 1067174727.6800004, + "range": 703410176.0, + "samples": 266, + "stdev": 127203616.38973977, + "sum": 115527344128.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6417.066666666667, + "max": 59507838.86666667, + "mean": 1324360.28320802, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3487919.1617161715, + "non_zero_median": 12014.933333333332, + "percentile25": 0.0, + "percentile75": 6417.066666666667, + "percentile90": 29491.200000000004, + "percentile99": 57750491.02666666, + "percentile999": 59418253.88533334, + "range": 59507838.86666667, + "samples": 266, + "stdev": 8661857.717339447, + "sum": 352279835.3333332 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 417.78995, + "mean": 6.874349338345865, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 44.59943717073171, + "non_zero_median": 0.042951, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.007464500000000001, + "percentile99": 319.5478185000024, + "percentile999": 413.8150048550002, + "range": 417.78995, + "samples": 266, + "stdev": 47.565667000873354, + "sum": 1828.5769239999997 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.3395145, + "max": 432.789787, + "mean": 21.589351244360902, + "median": 0.0, + "min": -0.000658, + "non_zero_mean": 51.27470920535714, + "non_zero_median": 0.7214685000000001, + "percentile25": 0.0, + "percentile75": 0.3395145, + "percentile90": 57.660959, + "percentile99": 348.04346635000036, + "percentile999": 416.8892408350008, + "range": 432.790445, + "samples": 266, + "stdev": 69.25894952618773, + "sum": 5742.767431 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.933333333333334, + "mean": 0.143609022556391, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.9384615384615387, + "non_zero_median": 3.2666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.8366666666666864, + "percentile999": 4.827333333333339, + "range": 4.933333333333334, + "samples": 266, + "stdev": 0.6949072658959725, + "sum": 38.199999999999996 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9998.733333333334, + "mean": 423.7558897243108, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8670.697435897437, + "non_zero_median": 8957.866666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9944.686666666666, + "percentile999": 9996.701666666668, + "range": 9998.733333333334, + "samples": 266, + "stdev": 1893.8110730048077, + "sum": 112719.06666666668 + }, + "pg_stat_database_blks_hit": { + "iqr": 23459.05, + "max": 209986.2, + "mean": 25531.769423558897, + "median": 25591.36666666667, + "min": 2413.133333333333, + "non_zero_mean": 25531.769423558897, + "non_zero_median": 25591.36666666667, + "percentile25": 9830.916666666668, + "percentile75": 33289.96666666667, + "percentile90": 44471.33333333333, + "percentile99": 74766.77000000035, + "percentile999": 197686.89633333398, + "range": 207573.06666666668, + "samples": 266, + "stdev": 20626.102413423643, + "sum": 6791450.666666665 + }, + "pg_stat_database_blks_read": { + "iqr": 0.6, + "max": 214.53333333333333, + "mean": 1.3779448621553885, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7371248025276462, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6666666666666666, + "percentile90": 1.7000000000000002, + "percentile99": 4.026666666666673, + "percentile999": 159.51933333333616, + "range": 214.53333333333333, + "samples": 266, + "stdev": 13.15080655295236, + "sum": 366.5333333333329 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 266, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.6, + "max": 214.53333333333333, + "mean": 1.3779448621553885, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7371248025276462, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6666666666666666, + "percentile90": 1.7000000000000002, + "percentile99": 4.026666666666673, + "percentile999": 159.51933333333616, + "range": 214.53333333333333, + "samples": 266, + "stdev": 13.15080655295236, + "sum": 366.5333333333329 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.3333333333333335, + "max": 4001.733333333333, + "mean": 40.80426065162907, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 100.49938271604938, + "non_zero_median": 13.133333333333333, + "percentile25": 0.0, + "percentile75": 2.3333333333333335, + "percentile90": 58.8, + "percentile99": 89.92333333333382, + "percentile999": 4001.4153333333334, + "range": 4001.733333333333, + "samples": 266, + "stdev": 346.0684705802982, + "sum": 10853.933333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 12016.95, + "max": 474386.73333333334, + "mean": 14904.10426065163, + "median": 13727.933333333334, + "min": 1362.3333333333333, + "non_zero_mean": 14904.10426065163, + "non_zero_median": 13727.933333333334, + "percentile25": 5102.316666666667, + "percentile75": 17119.266666666666, + "percentile90": 25835.86666666667, + "percentile99": 35282.640000000014, + "percentile999": 358811.77100000595, + "range": 473024.4, + "samples": 266, + "stdev": 29576.820647195458, + "sum": 3964491.7333333343 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.866666666666667, + "max": 8005.333333333333, + "mean": 45.719799498746866, + "median": 2.6, + "min": 0.0, + "non_zero_mean": 55.029260935143284, + "non_zero_median": 4.2, + "percentile25": 1.2666666666666666, + "percentile75": 10.133333333333333, + "percentile90": 59.03333333333333, + "percentile99": 142.43333333333334, + "percentile999": 5921.832666666774, + "range": 8005.333333333333, + "samples": 266, + "stdev": 490.9567178494139, + "sum": 12161.466666666665 + }, + "pg_stat_database_tup_returned": { + "iqr": 36902.55, + "max": 540431.7333333333, + "mean": 44518.512280701754, + "median": 37396.46666666667, + "min": 3236.866666666667, + "non_zero_mean": 44518.512280701754, + "non_zero_median": 37396.46666666667, + "percentile25": 14234.533333333333, + "percentile75": 51137.083333333336, + "percentile90": 87225.56666666667, + "percentile99": 162183.4900000006, + "percentile999": 537532.7570000001, + "range": 537194.8666666666, + "samples": 266, + "stdev": 53765.06537508824, + "sum": 11841924.266666666 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.566666666666666, + "max": 73.46666666666667, + "mean": 10.260902255639097, + "median": 6.8, + "min": 0.13333333333333333, + "non_zero_mean": 10.260902255639097, + "non_zero_median": 6.8, + "percentile25": 3.3, + "percentile75": 15.866666666666667, + "percentile90": 24.266666666666666, + "percentile99": 39.26333333333345, + "percentile999": 67.30100000000031, + "range": 73.33333333333333, + "samples": 266, + "stdev": 10.028011929088875, + "sum": 2729.4 + }, + "pg_stat_database_xact_commit": { + "iqr": 121.73333333333335, + "max": 710.5333333333333, + "mean": 121.98471177944862, + "median": 89.36666666666667, + "min": 4.866666666666666, + "non_zero_mean": 121.98471177944862, + "non_zero_median": 89.36666666666667, + "percentile25": 37.06666666666666, + "percentile75": 158.8, + "percentile90": 274.93333333333334, + "percentile99": 565.5666666666668, + "percentile999": 691.7183333333343, + "range": 705.6666666666666, + "samples": 266, + "stdev": 124.47318783880804, + "sum": 32447.93333333332 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.12581453634085213, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.12581453634085213, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 266, + "stdev": 0.14389703641037146, + "sum": 33.46666666666668 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 266, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 23.766666666666676, + "max": 149.33333333333331, + "mean": 39.169172932330824, + "median": 32.2, + "min": 1.4666666666666666, + "non_zero_mean": 39.169172932330824, + "non_zero_median": 32.2, + "percentile25": 23.333333333333332, + "percentile75": 47.10000000000001, + "percentile90": 72.30000000000001, + "percentile99": 118.11333333333343, + "percentile999": 143.62700000000027, + "range": 147.86666666666665, + "samples": 266, + "stdev": 24.17784634025538, + "sum": 10418.999999999996 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2022-01-10T16:47:34.154306+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2022-01-10T14_44_55_783_0000", + "ended": "2022-01-10T17:53:57.640988+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.01733225350039902, + "iterations": 1, + "max": 0.5642474340002082, + "mean": 0.5438460174665427, + "median": 0.5453291899993928, + "min": 0.5218645260001722, + "rounds": 15, + "stddev": 0.0121092956458406 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.022319644499475544, + "iterations": 1, + "max": 1.3351531769994835, + "mean": 0.6022373179999704, + "median": 0.5539277190000576, + "min": 0.5251991369996176, + "rounds": 15, + "stddev": 0.20320058066736874 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.004829905000406143, + "iterations": 1, + "max": 0.4177414929999941, + "mean": 0.4049160666998432, + "median": 0.4030999644996882, + "min": 0.39958180600024207, + "rounds": 10, + "stddev": 0.006116036158365749 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.00936185199952888, + "iterations": 1, + "max": 1.031987860999834, + "mean": 0.2715037814000516, + "median": 0.2607283385000301, + "min": 0.24655108699971606, + "rounds": 100, + "stddev": 0.07848786759478764 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.013887279500067962, + "iterations": 1, + "max": 0.5313930150005035, + "mean": 0.46687894545000747, + "median": 0.4664714859995911, + "min": 0.4415862020005079, + "rounds": 100, + "stddev": 0.01300627478742509 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.0609244179995585, + "iterations": 1, + "max": 1.4921608659997219, + "mean": 1.347095959299986, + "median": 1.326068761499755, + "min": 1.2790120129993738, + "rounds": 10, + "stddev": 0.06293879422882348 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0029742629994871095, + "iterations": 1, + "max": 0.0786539119999361, + "mean": 0.07324130899982825, + "median": 0.07240530000035506, + "min": 0.07074859599924821, + "rounds": 13, + "stddev": 0.0023138677797536152 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0028670009996858425, + "iterations": 1, + "max": 0.07852891600032308, + "mean": 0.07547998857158486, + "median": 0.07525545550060997, + "min": 0.07145681600013631, + "rounds": 14, + "stddev": 0.0018959568072077124 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.003687148000608431, + "iterations": 1, + "max": 0.11536316199999419, + "mean": 0.10820083219987282, + "median": 0.11038728699986677, + "min": 0.0911587700002201, + "rounds": 10, + "stddev": 0.007426399529994481 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.007684619000428938, + "iterations": 1, + "max": 0.1272969629999352, + "mean": 0.11442678499988687, + "median": 0.1136659150001833, + "min": 0.10732553399975586, + "rounds": 10, + "stddev": 0.005938387410102994 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.010557089999565505, + "iterations": 1, + "max": 0.2852156710005147, + "mean": 0.27637345599987384, + "median": 0.2767830474999755, + "min": 0.26713004399971396, + "rounds": 10, + "stddev": 0.006305086072568478 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.0035806280011456693, + "iterations": 1, + "max": 0.11638002600011532, + "mean": 0.11134780140000658, + "median": 0.11081335049993868, + "min": 0.10755374899963499, + "rounds": 10, + "stddev": 0.002757607329991348 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0025883480002448778, + "iterations": 1, + "max": 0.07729474600000685, + "mean": 0.0738780914284689, + "median": 0.0738500124998609, + "min": 0.06877768499998638, + "rounds": 14, + "stddev": 0.0021684631363738578 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.03011239200077398, + "iterations": 1, + "max": 0.7493601500000295, + "mean": 0.636477565899986, + "median": 0.6191920015003234, + "min": 0.6074878439994791, + "rounds": 10, + "stddev": 0.04204907459052402 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0019004329999461334, + "iterations": 1, + "max": 0.09852131899970118, + "mean": 0.09515238063630188, + "median": 0.09492796599988651, + "min": 0.09366235900051834, + "rounds": 11, + "stddev": 0.0014225133897013624 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.0027199870000913506, + "iterations": 1, + "max": 0.10962375500002963, + "mean": 0.10611069529995802, + "median": 0.10627685899999051, + "min": 0.10279122599968105, + "rounds": 10, + "stddev": 0.0023479409745288046 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.005678348999936134, + "iterations": 1, + "max": 0.18286455999987083, + "mean": 0.17689154279996727, + "median": 0.17681780850034556, + "min": 0.17287997999937943, + "rounds": 10, + "stddev": 0.0034341773827568866 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.003135841749553947, + "iterations": 1, + "max": 0.07807749699986744, + "mean": 0.07147004046655639, + "median": 0.07080119100010052, + "min": 0.06795227600014186, + "rounds": 15, + "stddev": 0.002558354563641923 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.003334738999910769, + "iterations": 1, + "max": 0.14884815000004892, + "mean": 0.0799032565712748, + "median": 0.07417685349946623, + "min": 0.07164648999969359, + "rounds": 14, + "stddev": 0.019977862050599655 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.021584892000646505, + "iterations": 1, + "max": 0.47719721300018136, + "mean": 0.3816251994000595, + "median": 0.37192017600000327, + "min": 0.3555690200000754, + "rounds": 10, + "stddev": 0.0353269118745799 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.01252333800039196, + "iterations": 1, + "max": 0.3809552779994192, + "mean": 0.37348765809992984, + "median": 0.3763619119995383, + "min": 0.3616437330001645, + "rounds": 10, + "stddev": 0.007429817223413595 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.00420308850038964, + "iterations": 1, + "max": 0.10120343800008413, + "mean": 0.09494277036372462, + "median": 0.093813926000621, + "min": 0.09087540499967872, + "rounds": 11, + "stddev": 0.0032053722644276293 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.0014070929996705672, + "iterations": 1, + "max": 0.08600932099943748, + "mean": 0.08346249758331699, + "median": 0.08391584650007644, + "min": 0.07938606100015022, + "rounds": 12, + "stddev": 0.0017398966618100609 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.003924707999431121, + "iterations": 1, + "max": 0.18376122399968153, + "mean": 0.17614257809991613, + "median": 0.17713760449987603, + "min": 0.1665855860001102, + "rounds": 10, + "stddev": 0.0050902062161142505 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0019356130003416183, + "iterations": 1, + "max": 0.07202332199994999, + "mean": 0.07000518699993942, + "median": 0.07038984700011497, + "min": 0.06825061700055812, + "rounds": 15, + "stddev": 0.0012195074625827018 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.009220257999913883, + "iterations": 1, + "max": 0.21480425699974148, + "mean": 0.20403609659988434, + "median": 0.20234002349980074, + "min": 0.19577859200035164, + "rounds": 10, + "stddev": 0.00649579044267798 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.005719774499539199, + "iterations": 1, + "max": 0.08817253300003358, + "mean": 0.08124472616661176, + "median": 0.08056407099957141, + "min": 0.07713642399994569, + "rounds": 12, + "stddev": 0.003516667708223852 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.008359072999155615, + "iterations": 1, + "max": 0.3602956439999616, + "mean": 0.3515644068999791, + "median": 0.35083766300022035, + "min": 0.3418028919995777, + "rounds": 10, + "stddev": 0.005996793001114188 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.006818271000156528, + "iterations": 1, + "max": 0.23531410799932928, + "mean": 0.22960956339984478, + "median": 0.22963783249997505, + "min": 0.22380169699954422, + "rounds": 10, + "stddev": 0.00420365248597677 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.005064789999778441, + "iterations": 1, + "max": 0.1719087829997079, + "mean": 0.16119169489984414, + "median": 0.16076481949949084, + "min": 0.15444329899946752, + "rounds": 10, + "stddev": 0.005052310392858246 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.11899825699947542, + "iterations": 1, + "max": 0.2370095729993409, + "mean": 0.1583895812999799, + "median": 0.1112712684998769, + "min": 0.10699378800018167, + "rounds": 10, + "stddev": 0.06298242341026304 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.003442562249574621, + "iterations": 1, + "max": 0.07196036100049241, + "mean": 0.06964340599994709, + "median": 0.06889227800002118, + "min": 0.06743917199946736, + "rounds": 15, + "stddev": 0.0017132999743026773 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.003136353250056345, + "iterations": 1, + "max": 0.09712769900033891, + "mean": 0.09217757309091791, + "median": 0.09285617999921669, + "min": 0.08783944399965549, + "rounds": 11, + "stddev": 0.002899411941406656 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0016902710003705579, + "iterations": 1, + "max": 0.0812767610004812, + "mean": 0.07455147214276232, + "median": 0.07426745899965681, + "min": 0.07077102099992771, + "rounds": 14, + "stddev": 0.0025578660950977 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.005977073000394739, + "iterations": 1, + "max": 0.22159732000000076, + "mean": 0.13198693440008355, + "median": 0.12323932749995947, + "min": 0.11241966000034154, + "rounds": 10, + "stddev": 0.03176915558683942 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.001918657999340212, + "iterations": 1, + "max": 0.07633008000084374, + "mean": 0.07303408714278703, + "median": 0.07272026649980035, + "min": 0.070959681000204, + "rounds": 14, + "stddev": 0.0016028772821674196 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.024842980000357784, + "iterations": 1, + "max": 0.6067280129991559, + "mean": 0.5846249602000171, + "median": 0.587729757499801, + "min": 0.5605169070004195, + "rounds": 10, + "stddev": 0.014881934697797007 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.02158495099956781, + "iterations": 1, + "max": 0.7597864780000236, + "mean": 0.6569714500001282, + "median": 0.6392881665001369, + "min": 0.6215516920001392, + "rounds": 10, + "stddev": 0.04913345674927488 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.020544437000353355, + "iterations": 1, + "max": 0.2922048749996975, + "mean": 0.27568163050000294, + "median": 0.27502870150010494, + "min": 0.24925487300060922, + "rounds": 10, + "stddev": 0.013182063120786229 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.005316076000781322, + "iterations": 1, + "max": 0.21631655399960437, + "mean": 0.08459288228570065, + "median": 0.07368587299970386, + "min": 0.07148600300024555, + "rounds": 14, + "stddev": 0.0380115747580388 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.001913518000037584, + "iterations": 1, + "max": 0.07516117900013342, + "mean": 0.07276191414296461, + "median": 0.07277253250003923, + "min": 0.0698051059998761, + "rounds": 14, + "stddev": 0.0014849834990441286 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0015875259996391833, + "iterations": 1, + "max": 0.07639645100061898, + "mean": 0.07381142150006781, + "median": 0.07390925150048133, + "min": 0.0711338720002459, + "rounds": 14, + "stddev": 0.0015463358771952824 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0044500589999643125, + "iterations": 1, + "max": 0.0819732389991259, + "mean": 0.07686836792324286, + "median": 0.07700907200069196, + "min": 0.07302811700083112, + "rounds": 13, + "stddev": 0.003037841034616278 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.008648883999740065, + "iterations": 1, + "max": 0.319896802000585, + "mean": 0.29576076200009993, + "median": 0.29517814950031607, + "min": 0.2738748890005809, + "rounds": 10, + "stddev": 0.011842402172290612 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.006009091999658267, + "iterations": 1, + "max": 0.23194674799924542, + "mean": 0.22650836070006336, + "median": 0.22679994600002829, + "min": 0.21879064699987794, + "rounds": 10, + "stddev": 0.0039297143262202955 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.009110028000577586, + "iterations": 1, + "max": 0.9357714730003863, + "mean": 0.8959326440999575, + "median": 0.9211175784998886, + "min": 0.6551154130002033, + "rounds": 10, + "stddev": 0.08487968704157624 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.158752836999156, + "iterations": 1, + "max": 8.829870391999975, + "mean": 8.699841324000044, + "median": 8.669080131499868, + "min": 8.601760561999981, + "rounds": 10, + "stddev": 0.08719740280959248 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.032141613999556284, + "iterations": 1, + "max": 1.0919087380007113, + "mean": 1.0643205014002888, + "median": 1.0653345884998089, + "min": 1.0314897060006842, + "rounds": 10, + "stddev": 0.020428991120497043 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.2967039909999585, + "iterations": 1, + "max": 9.373549473999446, + "mean": 9.054734087000089, + "median": 8.98143425950002, + "min": 8.892106097000578, + "rounds": 10, + "stddev": 0.1716659036252197 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.5189236019996315, + "iterations": 1, + "max": 1.5795622900004673, + "mean": 1.2624983184000484, + "median": 1.1112256390001676, + "min": 1.006186298999637, + "rounds": 10, + "stddev": 0.26435800190107867 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.021764428000096814, + "iterations": 1, + "max": 0.41684054200050014, + "mean": 0.4010302881001735, + "median": 0.40086950249997244, + "min": 0.3789510760007033, + "rounds": 10, + "stddev": 0.012852511933172052 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.08941948299798241, + "iterations": 1, + "max": 11.459284219001347, + "mean": 11.325147223699968, + "median": 11.317509578499994, + "min": 11.237588306999896, + "rounds": 10, + "stddev": 0.06906298553083273 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.01068128899896692, + "iterations": 1, + "max": 0.4245971529999224, + "mean": 0.41020659610012444, + "median": 0.4094100470001649, + "min": 0.39840869900035614, + "rounds": 10, + "stddev": 0.008043240430034545 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.2916373170000952, + "iterations": 1, + "max": 0.8603243019997535, + "mean": 0.6280426007999267, + "median": 0.6366782274999423, + "min": 0.437165121000362, + "rounds": 10, + "stddev": 0.158276459069627 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.15659905199936475, + "iterations": 1, + "max": 11.918877838999833, + "mean": 11.549199110599876, + "median": 11.53474438000012, + "min": 11.260443500999827, + "rounds": 10, + "stddev": 0.19269624354918874 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.16596589999790012, + "iterations": 1, + "max": 8.04647786500027, + "mean": 7.78448443940033, + "median": 7.7112770800004, + "min": 7.653914051999891, + "rounds": 10, + "stddev": 0.15065236405716004 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 259.22914460299944, + "mean": 259.22914460299944, + "median": 259.22914460299944, + "min": 259.22914460299944, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 415.75274521999927, + "mean": 415.75274521999927, + "median": 415.75274521999927, + "min": 415.75274521999927, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.010959041999740293, + "iterations": 1, + "max": 0.2710311869996076, + "mean": 0.2606046941000386, + "median": 0.2599049710001964, + "min": 0.24821695999980875, + "rounds": 10, + "stddev": 0.007425584298807042 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.0264178389998051, + "iterations": 1, + "max": 0.7530124869999781, + "mean": 0.6526574383998195, + "median": 0.6423165284995775, + "min": 0.6200921519994154, + "rounds": 10, + "stddev": 0.038757539261238316 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.10889771100119106, + "iterations": 1, + "max": 9.895409524999195, + "mean": 9.626246524199633, + "median": 9.62490184499893, + "min": 9.376768244999766, + "rounds": 10, + "stddev": 0.1326357230874952 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.3550728589998471, + "iterations": 1, + "max": 3.737129920999905, + "mean": 3.139607579100084, + "median": 3.1554504675004864, + "min": 2.5276592999998684, + "rounds": 10, + "stddev": 0.3674989290263973 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.13241749000007985, + "iterations": 1, + "max": 2.8648435800005245, + "mean": 1.9997503001998667, + "median": 1.9603875694997441, + "min": 1.4430213149998963, + "rounds": 10, + "stddev": 0.3489900523587252 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.4677515139992465, + "iterations": 1, + "max": 26.137869789999968, + "mean": 24.950252325499967, + "median": 24.795017229499535, + "min": 24.424340929999744, + "rounds": 10, + "stddev": 0.5123450857591079 + } + }, + "started": "2022-01-10T16:38:31.748304+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-chatty.json new file mode 100644 index 00000000..0b28c47f --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-chatty.json @@ -0,0 +1,636 @@ +{ + "ended": "2022-01-10T15:50:03.355151+00:00", + "id": "run-2022-01-10T14_44_55_783_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 13026.066666666666, + "max": 43384.73333333334, + "mean": 31989.922222222223, + "median": 38919.23333333334, + "min": 1751.4666666666667, + "non_zero_mean": 31989.922222222223, + "non_zero_median": 38919.23333333334, + "percentile25": 26582.75, + "percentile75": 39608.816666666666, + "percentile90": 40796.066666666666, + "percentile99": 42831.67, + "percentile999": 43329.427, + "range": 41633.26666666667, + "samples": 66, + "stdev": 12755.442012928199, + "sum": 2111334.866666666 + } + }, + "cpu": { + "system": { + "iqr": 0.20983333333333332, + "max": 1.056000000000002, + "mean": 0.8387373737373738, + "median": 0.9963333333333342, + "min": 0.04066666666666711, + "non_zero_mean": 0.8387373737373738, + "non_zero_median": 0.9963333333333342, + "percentile25": 0.806, + "percentile75": 1.0158333333333334, + "percentile90": 1.035333333333333, + "percentile99": 1.0559999999999996, + "percentile999": 1.0560000000000018, + "range": 1.015333333333335, + "samples": 66, + "stdev": 0.3028302535166565, + "sum": 55.35666666666666 + }, + "user": { + "iqr": 2.102333333333334, + "max": 6.529999999999996, + "mean": 5.054505050505051, + "median": 6.3100000000000005, + "min": 0.19866666666666408, + "non_zero_mean": 5.054505050505051, + "non_zero_median": 6.3100000000000005, + "percentile25": 4.27516666666667, + "percentile75": 6.377500000000004, + "percentile90": 6.424000000000001, + "percentile99": 6.516566666666665, + "percentile999": 6.528656666666663, + "range": 6.3313333333333315, + "samples": 66, + "stdev": 2.0317689116212274, + "sum": 333.5973333333334 + } + }, + "entropy_available_bits": { + "iqr": 9.85, + "max": 216.73333333333335, + "mean": 26.81919191919192, + "median": 6.266666666666667, + "min": 0.0, + "non_zero_mean": 36.12380952380953, + "non_zero_median": 7.733333333333333, + "percentile25": 0.16666666666666666, + "percentile75": 10.016666666666666, + "percentile90": 121.36666666666667, + "percentile99": 213.61333333333332, + "percentile999": 216.42133333333337, + "range": 216.73333333333335, + "samples": 66, + "stdev": 60.97976435008305, + "sum": 1770.0666666666668 + }, + "memory": { + "used": { + "iqr": 1505569792.0, + "max": 7453126656.0, + "mean": 6352255658.666667, + "median": 6825371648.0, + "min": 3471736832.0, + "non_zero_mean": 6352255658.666667, + "non_zero_median": 6825371648.0, + "percentile25": 5602450432.0, + "percentile75": 7108020224.0, + "percentile90": 7216924672.0, + "percentile99": 7367064575.999999, + "percentile999": 7444520448.0, + "range": 3981389824.0, + "samples": 66, + "stdev": 954241682.0460553, + "sum": 419248873472.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.0666666666666667, + "mean": 0.03838383838383838, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6333333333333333, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.0666666666666667, + "percentile999": 1.0666666666666667, + "range": 1.0666666666666667, + "samples": 66, + "stdev": 0.1867621023657714, + "sum": 2.533333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 2341.0000000000005, + "max": 4815.333333333334, + "mean": 3480.708080808081, + "median": 4498.166666666667, + "min": 4.133333333333334, + "non_zero_mean": 3480.708080808081, + "non_zero_median": 4498.166666666667, + "percentile25": 2313.683333333333, + "percentile75": 4654.683333333333, + "percentile90": 4703.466666666667, + "percentile99": 4801.8133333333335, + "percentile999": 4813.981333333334, + "range": 4811.200000000001, + "samples": 66, + "stdev": 1646.7249995591287, + "sum": 229726.73333333325 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 175.8166666666666, + "max": 2210.5333333333333, + "mean": 1619.8050505050505, + "median": 1710.7666666666667, + "min": 673.1333333333333, + "non_zero_mean": 1619.8050505050505, + "non_zero_median": 1710.7666666666667, + "percentile25": 1606.65, + "percentile75": 1782.4666666666667, + "percentile90": 1960.7, + "percentile99": 2192.2466666666664, + "percentile999": 2208.7046666666665, + "range": 1537.4, + "samples": 66, + "stdev": 352.8160719463792, + "sum": 106907.1333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.04466666666666633, + "max": 0.23266666666666633, + "mean": 0.14515151515151514, + "median": 0.15766666666666704, + "min": 0.026000000000000037, + "non_zero_mean": 0.14515151515151514, + "non_zero_median": 0.15766666666666704, + "percentile25": 0.12466666666666673, + "percentile75": 0.16933333333333306, + "percentile90": 0.19466666666666665, + "percentile99": 0.22789999999999966, + "percentile999": 0.23218999999999967, + "range": 0.20666666666666628, + "samples": 66, + "stdev": 0.04882629884813569, + "sum": 9.579999999999998 + }, + "user": { + "iqr": 0.031666666666667176, + "max": 0.5879999999999999, + "mean": 0.21690909090909088, + "median": 0.18000000000000016, + "min": 0.02933333333333318, + "non_zero_mean": 0.21690909090909088, + "non_zero_median": 0.18000000000000016, + "percentile25": 0.16366666666666632, + "percentile75": 0.1953333333333335, + "percentile90": 0.4926666666666667, + "percentile99": 0.5780333333333333, + "percentile999": 0.5870033333333332, + "range": 0.5586666666666666, + "samples": 66, + "stdev": 0.13860070602618058, + "sum": 14.315999999999999 + } + }, + "entropy_available_bits": { + "iqr": 1.5166666666666666, + "max": 191.86666666666667, + "mean": 11.755555555555556, + "median": 2.2, + "min": 0.4, + "non_zero_mean": 11.755555555555556, + "non_zero_median": 2.2, + "percentile25": 1.4, + "percentile75": 2.9166666666666665, + "percentile90": 4.966666666666667, + "percentile99": 184.10999999999993, + "percentile999": 191.09100000000004, + "range": 191.46666666666667, + "samples": 66, + "stdev": 38.47927950945744, + "sum": 775.8666666666667 + }, + "memory": { + "used": { + "iqr": 30151680.0, + "max": 474284032.0, + "mean": 443271354.1818182, + "median": 457650176.0, + "min": 368373760.0, + "non_zero_mean": 443271354.1818182, + "non_zero_median": 457650176.0, + "percentile25": 431954944.0, + "percentile75": 462106624.0, + "percentile90": 467271680.0, + "percentile99": 473570508.8, + "percentile999": 474212679.68, + "range": 105910272.0, + "samples": 66, + "stdev": 30168682.579252657, + "sum": 29255909376.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1874875.7333333334, + "max": 3356808.533333333, + "mean": 797139.5232323232, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2023508.0205128205, + "non_zero_median": 2218666.666666667, + "percentile25": 0.0, + "percentile75": 1874875.7333333334, + "percentile90": 2733943.466666667, + "percentile99": 3344029.013333333, + "percentile999": 3355530.581333333, + "range": 3356808.533333333, + "samples": 66, + "stdev": 1167503.41569467, + "sum": 52611208.53333334 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.029366, + "mean": 0.002666621212121212, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.014666416666666666, + "non_zero_median": 0.012524, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0096345, + "percentile99": 0.028937649999999995, + "percentile999": 0.029323165, + "range": 0.029366, + "samples": 66, + "stdev": 0.007026624502594841, + "sum": 0.17599700000000001 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 0.431885, + "mean": 0.01713389393939394, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.1130837, + "non_zero_median": 0.0371145, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0021665, + "percentile99": 0.3544699999999993, + "percentile999": 0.4241435000000003, + "range": 0.431885, + "samples": 66, + "stdev": 0.07024921016016507, + "sum": 1.1308370000000003 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 11.133333333333333, + "mean": 0.3797979797979798, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8.355555555555554, + "non_zero_median": 11.133333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 11.133333333333333, + "percentile999": 11.133333333333333, + "range": 11.133333333333333, + "samples": 66, + "stdev": 1.946286218424732, + "sum": 25.066666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9988.666666666666, + "mean": 453.57070707070704, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9978.555555555555, + "non_zero_median": 9983.733333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9985.46, + "percentile999": 9988.346, + "range": 9988.666666666666, + "samples": 66, + "stdev": 2094.451035927542, + "sum": 29935.666666666664 + }, + "pg_stat_database_blks_hit": { + "iqr": 4018.5666666666657, + "max": 36826.53333333333, + "mean": 21471.8898989899, + "median": 22724.533333333333, + "min": 4920.4, + "non_zero_mean": 21471.8898989899, + "non_zero_median": 22724.533333333333, + "percentile25": 19679.433333333334, + "percentile75": 23698.0, + "percentile90": 25025.033333333333, + "percentile99": 36783.89333333333, + "percentile999": 36822.26933333333, + "range": 31906.13333333333, + "samples": 66, + "stdev": 5867.454520390735, + "sum": 1417144.733333334 + }, + "pg_stat_database_blks_read": { + "iqr": 67.61666666666666, + "max": 148.13333333333333, + "mean": 97.27171717171717, + "median": 129.56666666666666, + "min": 0.0, + "non_zero_mean": 100.31145833333333, + "non_zero_median": 130.23333333333335, + "percentile25": 67.38333333333334, + "percentile75": 135.0, + "percentile90": 139.23333333333335, + "percentile99": 146.44333333333333, + "percentile999": 147.96433333333334, + "range": 148.13333333333333, + "samples": 66, + "stdev": 51.00947632503861, + "sum": 6419.933333333333 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 67.61666666666666, + "max": 148.13333333333333, + "mean": 97.27171717171717, + "median": 129.56666666666666, + "min": 0.0, + "non_zero_mean": 100.31145833333333, + "non_zero_median": 130.23333333333335, + "percentile25": 67.38333333333334, + "percentile75": 135.0, + "percentile90": 139.23333333333335, + "percentile99": 146.44333333333333, + "percentile999": 147.96433333333334, + "range": 148.13333333333333, + "samples": 66, + "stdev": 51.00947632503861, + "sum": 6419.933333333333 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.3333333333333333, + "mean": 0.18484848484848485, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7625, + "non_zero_median": 0.9, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9666666666666667, + "percentile99": 1.1599999999999984, + "percentile999": 1.3160000000000005, + "range": 1.3333333333333333, + "samples": 66, + "stdev": 0.369762308414696, + "sum": 12.200000000000001 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2107.366666666666, + "max": 23671.133333333335, + "mean": 7216.30101010101, + "median": 5598.633333333333, + "min": 3722.9333333333334, + "non_zero_mean": 7216.30101010101, + "non_zero_median": 5598.633333333333, + "percentile25": 5156.166666666667, + "percentile75": 7263.533333333333, + "percentile90": 11952.8, + "percentile99": 21668.699999999983, + "percentile999": 23470.89000000001, + "range": 19948.2, + "samples": 66, + "stdev": 3974.264598424413, + "sum": 476275.8666666667 + }, + "pg_stat_database_tup_inserted": { + "iqr": 355.31666666666666, + "max": 805.8, + "mean": 516.9131313131313, + "median": 690.1333333333334, + "min": 0.0, + "non_zero_mean": 524.865641025641, + "non_zero_median": 692.0666666666667, + "percentile25": 361.93333333333334, + "percentile75": 717.25, + "percentile90": 737.4666666666667, + "percentile99": 786.5166666666664, + "percentile999": 803.8716666666667, + "range": 805.8, + "samples": 66, + "stdev": 270.5112492963204, + "sum": 34116.26666666667 + }, + "pg_stat_database_tup_returned": { + "iqr": 2294.0499999999993, + "max": 29368.066666666666, + "mean": 9107.956565656566, + "median": 7023.9, + "min": 4640.733333333334, + "non_zero_mean": 9107.956565656566, + "non_zero_median": 7023.9, + "percentile25": 6416.783333333333, + "percentile75": 8710.833333333332, + "percentile90": 16205.533333333333, + "percentile99": 26817.683333333312, + "percentile999": 29113.028333333343, + "range": 24727.333333333332, + "samples": 66, + "stdev": 5105.941797285191, + "sum": 601125.1333333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 6.666666666666666, + "max": 123.0, + "mean": 9.865656565656565, + "median": 0.9666666666666667, + "min": 0.0, + "non_zero_mean": 11.226436781609195, + "non_zero_median": 1.4, + "percentile25": 0.15, + "percentile75": 6.816666666666666, + "percentile90": 27.96666666666667, + "percentile99": 113.03333333333325, + "percentile999": 122.00333333333337, + "range": 123.0, + "samples": 66, + "stdev": 24.4003768849252, + "sum": 651.1333333333332 + }, + "pg_stat_database_xact_commit": { + "iqr": 10.483333333333341, + "max": 135.66666666666666, + "mean": 58.5040404040404, + "median": 50.43333333333334, + "min": 9.133333333333333, + "non_zero_mean": 58.5040404040404, + "non_zero_median": 50.43333333333334, + "percentile25": 48.43333333333333, + "percentile75": 58.91666666666667, + "percentile90": 98.46666666666667, + "percentile99": 131.76666666666662, + "percentile999": 135.27666666666667, + "range": 126.53333333333333, + "samples": 66, + "stdev": 24.795296297215696, + "sum": 3861.2666666666664 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 4.400000000000004 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.8666666666666667, + "mean": 0.03333333333333333, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.1, + "non_zero_median": 1.1, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.8699999999999912, + "percentile999": 1.7670000000000035, + "range": 1.8666666666666667, + "samples": 66, + "stdev": 0.23278323433622572, + "sum": 2.2 + } + }, + "writes_completed": { + "total": { + "iqr": 26.400000000000006, + "max": 146.26666666666665, + "mean": 77.92626262626263, + "median": 76.63333333333333, + "min": 8.8, + "non_zero_mean": 77.92626262626263, + "non_zero_median": 76.63333333333333, + "percentile25": 67.16666666666666, + "percentile75": 93.56666666666666, + "percentile90": 115.2, + "percentile99": 140.06999999999994, + "percentile999": 145.64700000000002, + "range": 137.46666666666664, + "samples": 66, + "stdev": 28.9859267444729, + "sum": 5143.133333333333 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "5,3,4,6,7,8,9,11,10,14,18,19,20,21,22,23,24,25,26,27,28,30,31,29,33,32,34,35,36,37,38,40,43,39,42,41,44,45,46,47,48,49,53,52,50,54,51,55,57,56", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-10T15:30:55.214126Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2022-01-10T15:30:56.040489Z 2022-01-10T15:30:55.960819Z 2022-01-10T15:30:55.864026Z 2022-01-10T15:30:55.384202Z 2022-01-10T15:30:55.277462Z 2022-01-10T15:30:55.240358Z 2022-01-10T15:30:55.214126Z 2022-01-10T15:30:57.102143Z 2022-01-10T15:30:56.170233Z 2022-01-10T15:30:56.102909Z 2022-01-10T15:35:19.886521Z 2022-01-10T15:35:19.549053Z 2022-01-10T15:35:19.526782Z 2022-01-10T15:35:19.246670Z 2022-01-10T15:35:19.132232Z 2022-01-10T15:35:19.127251Z 2022-01-10T15:35:19.054083Z 2022-01-10T15:35:18.932108Z 2022-01-10T15:35:18.922821Z 2022-01-10T15:35:18.914102Z 2022-01-10T15:38:14.415697Z 2022-01-10T15:38:14.295001Z 2022-01-10T15:38:13.915180Z 2022-01-10T15:38:13.851055Z 2022-01-10T15:38:13.838530Z 2022-01-10T15:38:13.838354Z 2022-01-10T15:38:13.829683Z 2022-01-10T15:38:13.806627Z 2022-01-10T15:38:13.804676Z 2022-01-10T15:38:13.693967Z 2022-01-10T15:41:21.599413Z 2022-01-10T15:41:21.478627Z 2022-01-10T15:41:21.136370Z 2022-01-10T15:41:21.047108Z 2022-01-10T15:41:21.040385Z 2022-01-10T15:41:21.035465Z 2022-01-10T15:41:21.028856Z 2022-01-10T15:41:21.010908Z 2022-01-10T15:41:21.010145Z 2022-01-10T15:41:20.906805Z 2022-01-10T15:44:32.442725Z 2022-01-10T15:44:32.440612Z 2022-01-10T15:44:31.970075Z 2022-01-10T15:44:31.955180Z 2022-01-10T15:44:31.939595Z 2022-01-10T15:44:31.932085Z 2022-01-10T15:44:31.926895Z 2022-01-10T15:44:31.912722Z 2022-01-10T15:44:31.841729Z 2022-01-10T15:44:31.825805Z", + "created_diff": 0.9783388000000001, + "ended": "2022-01-10T15:47:14.069394Z", + "error_job_ids": " []", + "event_first": "2022-01-10T15:31:07.012595Z 2022-01-10T15:32:42.726732Z 2022-01-10T15:31:07.012595Z 2022-01-10T15:32:24.104462Z 2022-01-10T15:31:01.143330Z 2022-01-10T15:31:47.666855Z 2022-01-10T15:31:01.955963Z 2022-01-10T15:32:09.812592Z 2022-01-10T15:31:07.899383Z 2022-01-10T15:31:57.422953Z 2022-01-10T15:35:29.797655Z 2022-01-10T15:35:32.846844Z 2022-01-10T15:35:32.057807Z 2022-01-10T15:35:30.536896Z 2022-01-10T15:35:28.775634Z 2022-01-10T15:35:28.228965Z 2022-01-10T15:35:25.456715Z 2022-01-10T15:35:25.966998Z 2022-01-10T15:35:26.191590Z 2022-01-10T15:35:24.956753Z 2022-01-10T15:38:32.986570Z 2022-01-10T15:38:31.458982Z 2022-01-10T15:38:21.400606Z 2022-01-10T15:38:22.282757Z 2022-01-10T15:38:20.983960Z 2022-01-10T15:38:20.926028Z 2022-01-10T15:38:20.045506Z 2022-01-10T15:38:21.171019Z 2022-01-10T15:38:20.232092Z 2022-01-10T15:38:20.402946Z 2022-01-10T15:41:49.975517Z 2022-01-10T15:41:50.426293Z 2022-01-10T15:41:29.044899Z 2022-01-10T15:41:30.516773Z 2022-01-10T15:41:27.379061Z 2022-01-10T15:41:28.615818Z 2022-01-10T15:41:27.397821Z 2022-01-10T15:41:27.157167Z 2022-01-10T15:41:27.404045Z 2022-01-10T15:41:27.346561Z 2022-01-10T15:44:51.825494Z 2022-01-10T15:44:54.488805Z 2022-01-10T15:44:42.008775Z 2022-01-10T15:44:40.796649Z 2022-01-10T15:44:39.863672Z 2022-01-10T15:44:38.776091Z 2022-01-10T15:44:38.097293Z 2022-01-10T15:44:38.019997Z 2022-01-10T15:44:38.341621Z 2022-01-10T15:44:37.833769Z", + "event_last": "2022-01-10T15:33:42.209230Z 2022-01-10T15:34:51.288162Z 2022-01-10T15:33:42.339742Z 2022-01-10T15:34:41.091921Z 2022-01-10T15:33:34.752760Z 2022-01-10T15:34:17.708263Z 2022-01-10T15:33:38.004473Z 2022-01-10T15:34:32.119000Z 2022-01-10T15:33:44.393931Z 2022-01-10T15:34:24.554333Z 2022-01-10T15:37:52.998440Z 2022-01-10T15:37:57.875979Z 2022-01-10T15:37:54.783060Z 2022-01-10T15:37:54.478870Z 2022-01-10T15:37:52.012730Z 2022-01-10T15:37:52.480066Z 2022-01-10T15:37:50.167907Z 2022-01-10T15:37:50.743932Z 2022-01-10T15:37:50.167907Z 2022-01-10T15:37:49.002625Z 2022-01-10T15:40:54.275205Z 2022-01-10T15:40:53.965913Z 2022-01-10T15:40:48.496169Z 2022-01-10T15:40:46.338698Z 2022-01-10T15:40:48.496169Z 2022-01-10T15:40:46.665041Z 2022-01-10T15:40:48.549725Z 2022-01-10T15:40:46.665041Z 2022-01-10T15:40:44.961913Z 2022-01-10T15:40:46.288529Z 2022-01-10T15:44:06.071079Z 2022-01-10T15:44:06.803643Z 2022-01-10T15:43:52.831054Z 2022-01-10T15:43:54.761891Z 2022-01-10T15:43:52.049808Z 2022-01-10T15:43:53.788157Z 2022-01-10T15:43:51.473786Z 2022-01-10T15:43:53.009898Z 2022-01-10T15:43:51.835659Z 2022-01-10T15:43:51.255375Z 2022-01-10T15:47:10.331626Z 2022-01-10T15:47:14.069394Z 2022-01-10T15:47:04.947325Z 2022-01-10T15:47:03.294461Z 2022-01-10T15:47:04.217972Z 2022-01-10T15:47:03.652077Z 2022-01-10T15:47:02.548440Z 2022-01-10T15:47:02.378058Z 2022-01-10T15:47:02.548440Z 2022-01-10T15:47:01.930999Z", + "failed_job_ids": " []", + "finished": "2022-01-10T15:33:45.959457Z 2022-01-10T15:34:55.052764Z 2022-01-10T15:33:46.364792Z 2022-01-10T15:34:44.493432Z 2022-01-10T15:33:37.959404Z 2022-01-10T15:34:21.844056Z 2022-01-10T15:33:41.997274Z 2022-01-10T15:34:36.012264Z 2022-01-10T15:33:46.300423Z 2022-01-10T15:34:28.422793Z 2022-01-10T15:37:56.504207Z 2022-01-10T15:37:59.784025Z 2022-01-10T15:37:58.443674Z 2022-01-10T15:37:58.144232Z 2022-01-10T15:37:55.440123Z 2022-01-10T15:37:56.062132Z 2022-01-10T15:37:54.048774Z 2022-01-10T15:37:54.962309Z 2022-01-10T15:37:53.650930Z 2022-01-10T15:37:52.575463Z 2022-01-10T15:40:57.923975Z 2022-01-10T15:40:56.697137Z 2022-01-10T15:40:52.537964Z 2022-01-10T15:40:50.620868Z 2022-01-10T15:40:51.990935Z 2022-01-10T15:40:50.140596Z 2022-01-10T15:40:52.761688Z 2022-01-10T15:40:50.123340Z 2022-01-10T15:40:48.666883Z 2022-01-10T15:40:49.696247Z 2022-01-10T15:44:09.777191Z 2022-01-10T15:44:10.322878Z 2022-01-10T15:43:56.622108Z 2022-01-10T15:43:58.454647Z 2022-01-10T15:43:56.322381Z 2022-01-10T15:43:58.272586Z 2022-01-10T15:43:55.383177Z 2022-01-10T15:43:56.865874Z 2022-01-10T15:43:55.863070Z 2022-01-10T15:43:54.857813Z 2022-01-10T15:47:14.586370Z 2022-01-10T15:47:15.695524Z 2022-01-10T15:47:08.651415Z 2022-01-10T15:47:07.591561Z 2022-01-10T15:47:08.424411Z 2022-01-10T15:47:07.285581Z 2022-01-10T15:47:06.981571Z 2022-01-10T15:47:06.072010Z 2022-01-10T15:47:05.909139Z 2022-01-10T15:47:05.691644Z", + "finished_diff": 23.8055918, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 237.477014, + "mean": 161.90778892000006, + "min": 148.213939 + }, + "jobs_events_duration": { + "max": 156.494548, + "mean": 144.65789944, + "min": 128.56143 + }, + "jobs_events_lag": { + "max": -1.62613, + "mean": -3.6627647199999998, + "min": -4.484429 + }, + "jobs_waiting": { + "max": 19.963839, + "mean": 2.9601000200000005, + "min": 0.864219 + }, + "modified": "2022-01-10T15:30:57.178839Z 2022-01-10T15:30:57.143687Z 2022-01-10T15:30:57.108088Z 2022-01-10T15:30:55.874526Z 2022-01-10T15:30:55.843449Z 2022-01-10T15:30:55.810632Z 2022-01-10T15:30:55.772985Z 2022-01-10T15:31:15.360994Z 2022-01-10T15:30:57.273165Z 2022-01-10T15:30:57.219803Z 2022-01-10T15:35:20.961772Z 2022-01-10T15:35:20.906715Z 2022-01-10T15:35:20.854976Z 2022-01-10T15:35:20.813897Z 2022-01-10T15:35:19.670649Z 2022-01-10T15:35:19.631196Z 2022-01-10T15:35:19.590914Z 2022-01-10T15:35:19.546834Z 2022-01-10T15:35:19.510033Z 2022-01-10T15:35:19.474709Z 2022-01-10T15:38:19.277223Z 2022-01-10T15:38:19.190921Z 2022-01-10T15:38:14.530660Z 2022-01-10T15:38:14.498874Z 2022-01-10T15:38:14.468475Z 2022-01-10T15:38:14.436434Z 2022-01-10T15:38:14.394527Z 2022-01-10T15:38:14.359836Z 2022-01-10T15:38:14.310638Z 2022-01-10T15:38:14.269496Z 2022-01-10T15:41:40.947663Z 2022-01-10T15:41:40.904268Z 2022-01-10T15:41:21.722134Z 2022-01-10T15:41:21.687581Z 2022-01-10T15:41:21.654321Z 2022-01-10T15:41:21.622184Z 2022-01-10T15:41:21.586580Z 2022-01-10T15:41:21.551213Z 2022-01-10T15:41:21.520857Z 2022-01-10T15:41:21.487924Z 2022-01-10T15:44:42.842589Z 2022-01-10T15:44:42.627147Z 2022-01-10T15:44:32.668001Z 2022-01-10T15:44:32.636995Z 2022-01-10T15:44:32.604868Z 2022-01-10T15:44:32.571805Z 2022-01-10T15:44:32.534034Z 2022-01-10T15:44:32.501612Z 2022-01-10T15:44:32.470844Z 2022-01-10T15:44:32.429676Z", + "modified_diff": 11.1910902, + "started": "2022-01-10T15:30:57.661924Z 2022-01-10T15:30:57.575750Z 2022-01-10T15:30:57.460287Z 2022-01-10T15:30:56.387351Z 2022-01-10T15:30:56.236379Z 2022-01-10T15:30:56.211019Z 2022-01-10T15:30:56.078345Z 2022-01-10T15:31:15.559641Z 2022-01-10T15:30:57.894394Z 2022-01-10T15:30:57.751760Z 2022-01-10T15:35:21.594975Z 2022-01-10T15:35:21.558386Z 2022-01-10T15:35:21.354367Z 2022-01-10T15:35:21.143934Z 2022-01-10T15:35:20.385540Z 2022-01-10T15:35:20.222370Z 2022-01-10T15:35:20.135000Z 2022-01-10T15:35:20.029833Z 2022-01-10T15:35:19.944498Z 2022-01-10T15:35:19.854869Z 2022-01-10T15:38:19.639208Z 2022-01-10T15:38:19.536739Z 2022-01-10T15:38:15.426517Z 2022-01-10T15:38:15.366956Z 2022-01-10T15:38:15.235390Z 2022-01-10T15:38:15.152040Z 2022-01-10T15:38:15.045839Z 2022-01-10T15:38:14.963880Z 2022-01-10T15:38:14.874550Z 2022-01-10T15:38:14.805151Z 2022-01-10T15:41:41.563252Z 2022-01-10T15:41:41.396649Z 2022-01-10T15:41:22.573887Z 2022-01-10T15:41:22.471595Z 2022-01-10T15:41:22.370533Z 2022-01-10T15:41:22.290953Z 2022-01-10T15:41:22.212913Z 2022-01-10T15:41:22.130614Z 2022-01-10T15:41:22.058457Z 2022-01-10T15:41:21.983393Z 2022-01-10T15:44:43.828682Z 2022-01-10T15:44:43.530165Z 2022-01-10T15:44:33.618834Z 2022-01-10T15:44:33.491574Z 2022-01-10T15:44:33.353647Z 2022-01-10T15:44:33.270033Z 2022-01-10T15:44:33.160513Z 2022-01-10T15:44:33.087379Z 2022-01-10T15:44:33.010631Z 2022-01-10T15:44:32.933070Z", + "started_diff": 11.306186, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-10T15:50:00.639188+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-launching.json new file mode 100644 index 00000000..f6b26f80 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-10T14_44_55_783_0000-launching.json @@ -0,0 +1,634 @@ +{ + "ended": "2022-01-10T16:37:35.654536+00:00", + "id": "run-2022-01-10T14_44_55_783_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 16188.466666666669, + "max": 31063.733333333337, + "mean": 9278.215637860083, + "median": 4277.733333333334, + "min": 1526.1333333333332, + "non_zero_mean": 9278.215637860083, + "non_zero_median": 4277.733333333334, + "percentile25": 1768.4, + "percentile75": 17956.86666666667, + "percentile90": 22283.6, + "percentile99": 27858.82666666668, + "percentile999": 30743.242666666734, + "range": 29537.600000000006, + "samples": 81, + "stdev": 8968.345696846285, + "sum": 751535.4666666667 + } + }, + "cpu": { + "system": { + "iqr": 1.3486666666666605, + "max": 1.7980000000000016, + "mean": 0.6336213991769547, + "median": 0.2640000000000005, + "min": 0.030000000000001137, + "non_zero_mean": 0.6336213991769547, + "non_zero_median": 0.2640000000000005, + "percentile25": 0.038666666666669396, + "percentile75": 1.3873333333333298, + "percentile90": 1.6306666666666652, + "percentile99": 1.7857333333333296, + "percentile999": 1.7967733333333347, + "range": 1.7680000000000005, + "samples": 81, + "stdev": 0.6805847562392503, + "sum": 51.32333333333335 + }, + "user": { + "iqr": 5.749999999999986, + "max": 6.550666666666696, + "mean": 2.6715884773662553, + "median": 1.0860000000000052, + "min": 0.08333333333335606, + "non_zero_mean": 2.6715884773662553, + "non_zero_median": 1.0860000000000052, + "percentile25": 0.3013333333333473, + "percentile75": 6.051333333333333, + "percentile90": 6.309333333333333, + "percentile99": 6.543200000000006, + "percentile999": 6.549920000000027, + "range": 6.46733333333334, + "samples": 81, + "stdev": 2.6913949141825695, + "sum": 216.39866666666668 + } + }, + "entropy_available_bits": { + "iqr": 4.666666666666667, + "max": 213.73333333333335, + "mean": 22.609876543209875, + "median": 0.8, + "min": 0.0, + "non_zero_mean": 34.554716981132074, + "non_zero_median": 3.866666666666667, + "percentile25": 0.0, + "percentile75": 4.666666666666667, + "percentile90": 7.333333333333333, + "percentile99": 213.14666666666668, + "percentile999": 213.6746666666667, + "range": 213.73333333333335, + "samples": 81, + "stdev": 62.929396161369354, + "sum": 1831.4000000000005 + }, + "memory": { + "used": { + "iqr": 4459180032.0, + "max": 16891928576.0, + "mean": 8434276984.098765, + "median": 6582951936.0, + "min": 5066190848.0, + "non_zero_mean": 8434276984.098765, + "non_zero_median": 6582951936.0, + "percentile25": 6102089728.0, + "percentile75": 10561269760.0, + "percentile90": 14770712576.0, + "percentile99": 16879545548.8, + "percentile999": 16890690273.28, + "range": 11825737728.0, + "samples": 81, + "stdev": 3572500948.5744095, + "sum": 683176435712.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 6.2, + "mean": 0.15473251028806584, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.5666666666666667, + "non_zero_median": 0.8, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.0533333333333443, + "percentile999": 5.885333333333396, + "range": 6.2, + "samples": 81, + "stdev": 0.7547499336052252, + "sum": 12.533333333333337 + } + }, + "writes_completed": { + "total": { + "iqr": 695.7333333333333, + "max": 1790.2, + "mean": 413.45432098765434, + "median": 79.93333333333334, + "min": 0.26666666666666666, + "non_zero_mean": 413.45432098765434, + "non_zero_median": 79.93333333333334, + "percentile25": 3.9333333333333336, + "percentile75": 699.6666666666666, + "percentile90": 1424.8, + "percentile99": 1637.9333333333338, + "percentile999": 1774.9733333333363, + "range": 1789.9333333333334, + "samples": 81, + "stdev": 552.9230401653372, + "sum": 33489.8 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2057.9333333333334, + "max": 3995.0666666666666, + "mean": 1626.1728395061727, + "median": 955.8, + "min": 460.46666666666664, + "non_zero_mean": 1626.1728395061727, + "non_zero_median": 955.8, + "percentile25": 680.2, + "percentile75": 2738.133333333333, + "percentile90": 3276.4, + "percentile99": 3724.3466666666673, + "percentile999": 3967.994666666672, + "range": 3534.6, + "samples": 81, + "stdev": 1121.8879440368848, + "sum": 131719.99999999997 + } + }, + "cpu": { + "system": { + "iqr": 0.18933333333333358, + "max": 0.4106666666666664, + "mean": 0.12246913580246914, + "median": 0.044666666666666306, + "min": 0.01266666666666557, + "non_zero_mean": 0.12246913580246914, + "non_zero_median": 0.044666666666666306, + "percentile25": 0.02599999999999909, + "percentile75": 0.21533333333333265, + "percentile90": 0.31200000000000044, + "percentile99": 0.38933333333333214, + "percentile999": 0.4085333333333334, + "range": 0.39800000000000085, + "samples": 81, + "stdev": 0.12394247968959202, + "sum": 9.920000000000003 + }, + "user": { + "iqr": 0.28533333333333244, + "max": 0.8060000000000003, + "mean": 0.19022222222222224, + "median": 0.05733333333333329, + "min": 0.013333333333332574, + "non_zero_mean": 0.19022222222222224, + "non_zero_median": 0.05733333333333329, + "percentile25": 0.03866666666666655, + "percentile75": 0.323999999999999, + "percentile90": 0.4593333333333324, + "percentile99": 0.8001333333333337, + "percentile999": 0.8054133333333338, + "range": 0.7926666666666677, + "samples": 81, + "stdev": 0.20665048323733048, + "sum": 15.407999999999998 + } + }, + "entropy_available_bits": { + "iqr": 3.0, + "max": 213.13333333333333, + "mean": 13.793415637860083, + "median": 0.3333333333333333, + "min": 0.0, + "non_zero_mean": 25.982945736434107, + "non_zero_median": 2.8666666666666667, + "percentile25": 0.0, + "percentile75": 3.0, + "percentile90": 4.8, + "percentile99": 212.28, + "percentile999": 213.048, + "range": 213.13333333333333, + "samples": 81, + "stdev": 49.60232589185743, + "sum": 1117.2666666666664 + }, + "memory": { + "used": { + "iqr": 342695936.0, + "max": 1151320064.0, + "mean": 557180599.308642, + "median": 407060480.0, + "min": 373624832.0, + "non_zero_mean": 557180599.308642, + "non_zero_median": 407060480.0, + "percentile25": 382328832.0, + "percentile75": 725024768.0, + "percentile90": 1008021504.0, + "percentile99": 1135073689.6000001, + "percentile999": 1149695426.5600004, + "range": 777695232.0, + "samples": 81, + "stdev": 245387678.13505334, + "sum": 45131628544.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 13653.333333333334, + "max": 56176425.53333333, + "mean": 708969.4411522633, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2392771.8638888886, + "non_zero_median": 56524.8, + "percentile25": 0.0, + "percentile75": 13653.333333333334, + "percentile90": 73181.86666666667, + "percentile99": 11327035.506666826, + "percentile999": 51691486.53066756, + "range": 56176425.53333333, + "samples": 81, + "stdev": 6240165.683161859, + "sum": 57426524.73333333 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.770922, + "mean": 0.022460777777777777, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.13994792307692308, + "non_zero_median": 0.022151, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.01622, + "percentile99": 0.5719156000000006, + "percentile999": 0.751021360000004, + "range": 0.770922, + "samples": 81, + "stdev": 0.10544320415346832, + "sum": 1.8193229999999998 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.187088, + "max": 7.385117, + "mean": 0.4938335925925926, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.212137, + "non_zero_median": 0.602069, + "percentile25": 0.0, + "percentile75": 0.187088, + "percentile90": 1.432535, + "percentile99": 5.142057800000008, + "percentile999": 7.160811080000045, + "range": 7.385117, + "samples": 81, + "stdev": 1.2343057882832436, + "sum": 40.000521000000006 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 2.7333333333333334, + "mean": 0.10370370370370371, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.1, + "non_zero_median": 2.033333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.3066666666666684, + "percentile999": 2.6906666666666754, + "range": 2.7333333333333334, + "samples": 81, + "stdev": 0.4674992572780791, + "sum": 8.399999999999999 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9991.866666666667, + "mean": 428.26831275720167, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8672.433333333334, + "non_zero_median": 8828.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9971.333333333334, + "percentile999": 9989.813333333334, + "range": 9991.866666666667, + "samples": 81, + "stdev": 1913.8538951252642, + "sum": 34689.73333333334 + }, + "pg_stat_database_blks_hit": { + "iqr": 21448.86666666667, + "max": 105089.2, + "mean": 20712.78353909465, + "median": 8335.533333333333, + "min": 1201.2, + "non_zero_mean": 20712.78353909465, + "non_zero_median": 8335.533333333333, + "percentile25": 7060.266666666666, + "percentile75": 28509.133333333335, + "percentile90": 44132.4, + "percentile99": 98604.9866666667, + "percentile999": 104440.7786666668, + "range": 103888.0, + "samples": 81, + "stdev": 23111.13122012741, + "sum": 1677735.466666666 + }, + "pg_stat_database_blks_read": { + "iqr": 3.2666666666666666, + "max": 13.533333333333333, + "mean": 2.0131687242798355, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.544927536231884, + "non_zero_median": 2.0, + "percentile25": 0.0, + "percentile75": 3.2666666666666666, + "percentile90": 7.133333333333334, + "percentile99": 10.866666666666676, + "percentile999": 13.266666666666719, + "range": 13.533333333333333, + "samples": 81, + "stdev": 3.1551090798732284, + "sum": 163.06666666666663 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 81, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.2666666666666666, + "max": 13.533333333333333, + "mean": 2.0131687242798355, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.544927536231884, + "non_zero_median": 2.0, + "percentile25": 0.0, + "percentile75": 3.2666666666666666, + "percentile90": 7.133333333333334, + "percentile99": 10.866666666666676, + "percentile999": 13.266666666666719, + "range": 13.533333333333333, + "samples": 81, + "stdev": 3.1551090798732284, + "sum": 163.06666666666663 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 2.3333333333333335, + "mean": 0.22469135802469137, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.0705882352941176, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 2.0666666666666678, + "percentile999": 2.306666666666672, + "range": 2.3333333333333335, + "samples": 81, + "stdev": 0.5209440623036053, + "sum": 18.200000000000003 + }, + "pg_stat_database_tup_fetched": { + "iqr": 12067.599999999999, + "max": 52820.4, + "mean": 11007.455144032921, + "median": 4324.8, + "min": 581.3333333333334, + "non_zero_mean": 11007.455144032921, + "non_zero_median": 4324.8, + "percentile25": 3746.8, + "percentile75": 15814.4, + "percentile90": 24939.666666666668, + "percentile99": 49886.373333333344, + "percentile999": 52526.99733333339, + "range": 52239.066666666666, + "samples": 81, + "stdev": 11989.748862084874, + "sum": 891603.8666666667 + }, + "pg_stat_database_tup_inserted": { + "iqr": 22.733333333333334, + "max": 102.06666666666666, + "mean": 15.708641975308643, + "median": 4.133333333333334, + "min": 0.0, + "non_zero_mean": 24.949019607843137, + "non_zero_median": 15.133333333333333, + "percentile25": 0.0, + "percentile75": 22.733333333333334, + "percentile90": 51.333333333333336, + "percentile99": 85.00000000000006, + "percentile999": 100.36000000000034, + "range": 102.06666666666666, + "samples": 81, + "stdev": 24.02231935431077, + "sum": 1272.4000000000003 + }, + "pg_stat_database_tup_returned": { + "iqr": 19080.333333333336, + "max": 81442.53333333334, + "mean": 17599.362139917695, + "median": 7206.6, + "min": 1655.7333333333333, + "non_zero_mean": 17599.362139917695, + "non_zero_median": 7206.6, + "percentile25": 6387.8, + "percentile75": 25468.133333333335, + "percentile90": 38157.8, + "percentile99": 78803.92000000001, + "percentile999": 81178.67200000006, + "range": 79786.8, + "samples": 81, + "stdev": 18487.240207775558, + "sum": 1425548.3333333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 31.6, + "max": 78.0, + "mean": 16.19753086419753, + "median": 6.066666666666666, + "min": 0.0, + "non_zero_mean": 19.014492753623188, + "non_zero_median": 7.933333333333334, + "percentile25": 0.26666666666666666, + "percentile75": 31.866666666666667, + "percentile90": 46.53333333333333, + "percentile99": 73.30666666666669, + "percentile999": 77.53066666666676, + "range": 78.0, + "samples": 81, + "stdev": 21.180170087567088, + "sum": 1311.9999999999998 + }, + "pg_stat_database_xact_commit": { + "iqr": 254.26666666666668, + "max": 654.0, + "mean": 146.22551440329218, + "median": 38.0, + "min": 3.066666666666667, + "non_zero_mean": 146.22551440329218, + "non_zero_median": 38.0, + "percentile25": 9.533333333333333, + "percentile75": 263.8, + "percentile90": 390.53333333333336, + "percentile99": 613.7333333333335, + "percentile999": 649.9733333333342, + "range": 650.9333333333333, + "samples": 81, + "stdev": 175.13291605327183, + "sum": 11844.266666666665 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 81, + "stdev": 0.0, + "sum": 5.4 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.0032921810699588477, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.05333333333333409, + "percentile999": 0.24533333333333757, + "range": 0.26666666666666666, + "samples": 81, + "stdev": 0.02962962962962963, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 82.93333333333334, + "max": 170.13333333333333, + "mean": 53.41399176954732, + "median": 36.13333333333333, + "min": 1.1333333333333333, + "non_zero_mean": 53.41399176954732, + "non_zero_median": 36.13333333333333, + "percentile25": 12.0, + "percentile75": 94.93333333333334, + "percentile90": 128.26666666666665, + "percentile99": 166.98666666666668, + "percentile999": 169.81866666666673, + "range": 169.0, + "samples": 81, + "stdev": 50.31136354525948, + "sum": 4326.533333333334 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "60,65,62,64,70,66,61,68,69,67,63,71,78,75,74,76,73,84,77,82,86,80,81,83,89,88,87,90,92,85,91,94,95,97,98,103,107,99,100,108,105,102,111,113,110,116,115,112,109,114,119,122,117,124,126,127,129,120,121,128,123,131,125,134,132,133,141,138,130,135,139,136,118,147,148,152,143,142,146,151,156,157,159,145,164,150,166,149,153,162,170,172,169,168,173,165,171,174,175,176,185,177,184,191,178,183,182,180,179,181,187,186,196,190,189,193,194,199,192,195,188,197,200,201,205,204,202,203,198,206,208,207,212,210,211,209,214,213,215,216,218,217,220,219,227,221,223,222,226,228,229,231,225,224,232,233,234,235,230,237,236,243,238,242,244,245,239,247,248,249,250,246,251,253,255,259,241,258,240,256,263,254,268,260,262,252,269,257,261,266,265,270,264,267,271,275,272,273,274,276,279,278,277,282,280,283,288,281,287,292,284,286,289,291,296,302,297,290,285,294,308,293,298,301,295,299,303,310,300,311,304,313,306,309,307,314,312,315,316,318,317,321,305,323,322,319,324,329,320,327,332,326,333,335,328,334,337,330,325,342,344,331,338,336,340,343,339,351,346,353,348,347,341,354,349,357,352,358,356,359,345,350,363,355,365,364,369,368,366,362,367,371,370,361,360,373,374,372,375,376,382,378,385,379,381,384,386,392,380,377,388,389,390,383,396,394,397,391,387,395,405,398,403,400,402,393,399,404,408,407,412,411,406,401,414,409,410,419,417,418,421,420,413,424,415,416,423,427,425,426,430,431,422,428,433,429,436,432,435,442,434,443,438,447,450,439,445,441,451,444,446,452,448,454,458,449,453,455,457,456,437,460,464,462,459,461,440,463,465,467,468,472,466,470,473,469,474,471,475,476,477,478,482,486,483,479,480,484,487,495,489,491,481,485,490,499,492,507,488,493,500,498,496,502,494,504,505,497,501,503,506,511,508,512,509,514,513,517,515,519,516,518,510,520,521,526,534,524,523,530,532,522,537,525,527,528,536,535,529,533,540,543,538,531,542,547,541,539,550,546,545,544,548,559,560,549,562,552,557,564,556,555,553,558,565,563,561,566,568,571,554,567,572,569,551,570,573,575,574,576", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-10T15:51:53.106427Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2022-01-10T15:51:58.056599Z 2022-01-10T15:51:57.999659Z 2022-01-10T15:51:57.440825Z 2022-01-10T15:51:57.402030Z 2022-01-10T15:51:57.224287Z 2022-01-10T15:51:57.122148Z 2022-01-10T15:51:56.938255Z 2022-01-10T15:51:56.670407Z 2022-01-10T15:51:56.427260Z 2022-01-10T15:51:56.314598Z 2022-01-10T15:51:56.254154Z 2022-01-10T15:51:56.105674Z 2022-01-10T15:51:55.923562Z 2022-01-10T15:51:55.756990Z 2022-01-10T15:51:55.740685Z 2022-01-10T15:51:55.602661Z 2022-01-10T15:51:55.575794Z 2022-01-10T15:51:55.509507Z 2022-01-10T15:51:55.429539Z 2022-01-10T15:51:55.398784Z 2022-01-10T15:51:55.315381Z 2022-01-10T15:51:55.216008Z 2022-01-10T15:51:54.855478Z 2022-01-10T15:51:54.843284Z 2022-01-10T15:51:54.700410Z 2022-01-10T15:51:54.624154Z 2022-01-10T15:51:54.528481Z 2022-01-10T15:51:54.366957Z 2022-01-10T15:51:54.176299Z 2022-01-10T15:51:54.174391Z 2022-01-10T15:51:53.816428Z 2022-01-10T15:51:53.810011Z 2022-01-10T15:51:53.626957Z 2022-01-10T15:51:53.522274Z 2022-01-10T15:51:53.512326Z 2022-01-10T15:51:53.106427Z 2022-01-10T15:52:18.014983Z 2022-01-10T15:52:12.826371Z 2022-01-10T15:52:12.153817Z 2022-01-10T15:52:09.198991Z 2022-01-10T15:52:08.368451Z 2022-01-10T15:52:07.851094Z 2022-01-10T15:52:07.739880Z 2022-01-10T15:52:07.624750Z 2022-01-10T15:52:07.571567Z 2022-01-10T15:52:07.020527Z 2022-01-10T15:52:06.821388Z 2022-01-10T15:52:06.793607Z 2022-01-10T15:52:06.589791Z 2022-01-10T15:52:06.108921Z 2022-01-10T15:52:05.764167Z 2022-01-10T15:52:05.689165Z 2022-01-10T15:52:05.492695Z 2022-01-10T15:52:05.268572Z 2022-01-10T15:52:05.251330Z 2022-01-10T15:52:04.803824Z 2022-01-10T15:52:04.495983Z 2022-01-10T15:52:04.472597Z 2022-01-10T15:52:04.450068Z 2022-01-10T15:52:04.383492Z 2022-01-10T15:52:04.278490Z 2022-01-10T15:52:04.157140Z 2022-01-10T15:52:03.718483Z 2022-01-10T15:52:03.452758Z 2022-01-10T15:52:03.358068Z 2022-01-10T15:52:03.284441Z 2022-01-10T15:52:02.940096Z 2022-01-10T15:52:02.910018Z 2022-01-10T15:52:02.770241Z 2022-01-10T15:52:02.746400Z 2022-01-10T15:52:02.711201Z 2022-01-10T15:52:02.456273Z 2022-01-10T15:52:02.346527Z 2022-01-10T15:52:01.683548Z 2022-01-10T15:52:01.683562Z 2022-01-10T15:52:01.649455Z 2022-01-10T15:52:01.380747Z 2022-01-10T15:52:01.333180Z 2022-01-10T15:52:01.248591Z 2022-01-10T15:52:01.114369Z 2022-01-10T15:52:01.028176Z 2022-01-10T15:52:00.865453Z 2022-01-10T15:52:00.673544Z 2022-01-10T15:52:00.653633Z 2022-01-10T15:52:00.445155Z 2022-01-10T15:52:00.385011Z 2022-01-10T15:52:00.136952Z 2022-01-10T15:51:59.853528Z 2022-01-10T15:51:59.715366Z 2022-01-10T15:51:59.549243Z 2022-01-10T15:51:59.522707Z 2022-01-10T15:51:59.396246Z 2022-01-10T15:51:58.938704Z 2022-01-10T15:51:58.831574Z 2022-01-10T15:51:58.792853Z 2022-01-10T15:51:58.789592Z 2022-01-10T15:51:58.554119Z 2022-01-10T15:51:58.389919Z 2022-01-10T15:51:58.251034Z 2022-01-10T15:51:58.121565Z 2022-01-10T15:58:02.688370Z 2022-01-10T15:58:00.977874Z 2022-01-10T15:58:00.927830Z 2022-01-10T15:58:00.799208Z 2022-01-10T15:57:59.924059Z 2022-01-10T15:57:59.858889Z 2022-01-10T15:57:58.700633Z 2022-01-10T15:57:57.599237Z 2022-01-10T15:57:57.578185Z 2022-01-10T15:57:57.326493Z 2022-01-10T15:57:57.310964Z 2022-01-10T15:57:57.100378Z 2022-01-10T15:57:56.831967Z 2022-01-10T15:57:56.740420Z 2022-01-10T15:57:56.609833Z 2022-01-10T15:57:56.552829Z 2022-01-10T15:57:56.542428Z 2022-01-10T15:57:56.338177Z 2022-01-10T15:57:56.226074Z 2022-01-10T15:57:56.086940Z 2022-01-10T15:57:56.039345Z 2022-01-10T15:57:56.000728Z 2022-01-10T15:57:55.748368Z 2022-01-10T15:57:55.648720Z 2022-01-10T15:57:55.573653Z 2022-01-10T15:57:54.843853Z 2022-01-10T15:57:53.904584Z 2022-01-10T15:57:53.670310Z 2022-01-10T15:57:53.604344Z 2022-01-10T15:57:53.369387Z 2022-01-10T15:57:53.131878Z 2022-01-10T15:57:52.993454Z 2022-01-10T15:57:52.794786Z 2022-01-10T15:57:52.731088Z 2022-01-10T15:57:52.542181Z 2022-01-10T15:57:52.534621Z 2022-01-10T15:57:52.312030Z 2022-01-10T15:57:52.297545Z 2022-01-10T15:57:52.245101Z 2022-01-10T15:57:52.161394Z 2022-01-10T15:57:52.104994Z 2022-01-10T15:57:51.877602Z 2022-01-10T15:57:51.404378Z 2022-01-10T15:57:51.308378Z 2022-01-10T15:57:51.037704Z 2022-01-10T15:57:50.914650Z 2022-01-10T15:57:50.761749Z 2022-01-10T15:57:50.358464Z 2022-01-10T15:57:50.342328Z 2022-01-10T15:57:50.149892Z 2022-01-10T15:57:50.134850Z 2022-01-10T15:57:49.881689Z 2022-01-10T15:57:49.707513Z 2022-01-10T15:57:49.586778Z 2022-01-10T15:57:49.541164Z 2022-01-10T15:57:49.456884Z 2022-01-10T15:57:49.390419Z 2022-01-10T15:57:49.203534Z 2022-01-10T15:57:48.953598Z 2022-01-10T15:57:48.916379Z 2022-01-10T15:57:48.648057Z 2022-01-10T15:57:48.462379Z 2022-01-10T15:57:48.114962Z 2022-01-10T15:57:48.069101Z 2022-01-10T15:57:47.935196Z 2022-01-10T15:57:47.926249Z 2022-01-10T15:57:47.919524Z 2022-01-10T15:57:47.721768Z 2022-01-10T15:57:47.432198Z 2022-01-10T15:57:47.135757Z 2022-01-10T15:57:47.087274Z 2022-01-10T15:57:47.032113Z 2022-01-10T15:57:46.810645Z 2022-01-10T15:57:46.784029Z 2022-01-10T15:57:46.579546Z 2022-01-10T15:57:46.424688Z 2022-01-10T15:57:46.212667Z 2022-01-10T15:57:46.184740Z 2022-01-10T15:57:45.997879Z 2022-01-10T15:57:45.964369Z 2022-01-10T15:57:45.681761Z 2022-01-10T15:57:45.643907Z 2022-01-10T15:57:45.521632Z 2022-01-10T15:57:45.455137Z 2022-01-10T15:57:45.417922Z 2022-01-10T15:57:45.237468Z 2022-01-10T15:57:45.205600Z 2022-01-10T15:57:45.095650Z 2022-01-10T15:57:44.946906Z 2022-01-10T15:57:44.737640Z 2022-01-10T15:57:44.688903Z 2022-01-10T15:57:44.646782Z 2022-01-10T15:57:44.328624Z 2022-01-10T15:57:44.129427Z 2022-01-10T15:57:44.049913Z 2022-01-10T15:57:44.021955Z 2022-01-10T15:57:43.623408Z 2022-01-10T15:57:43.615235Z 2022-01-10T15:57:43.426736Z 2022-01-10T15:57:43.187144Z 2022-01-10T16:02:07.537468Z 2022-01-10T16:02:07.298060Z 2022-01-10T16:02:05.366589Z 2022-01-10T16:02:04.409409Z 2022-01-10T16:02:04.148743Z 2022-01-10T16:02:04.104242Z 2022-01-10T16:02:03.956132Z 2022-01-10T16:02:03.924837Z 2022-01-10T16:02:03.849943Z 2022-01-10T16:02:03.838007Z 2022-01-10T16:02:03.607163Z 2022-01-10T16:02:03.557484Z 2022-01-10T16:02:03.229566Z 2022-01-10T16:02:03.176356Z 2022-01-10T16:02:03.067378Z 2022-01-10T16:02:03.025258Z 2022-01-10T16:02:02.931642Z 2022-01-10T16:02:02.863451Z 2022-01-10T16:02:02.801008Z 2022-01-10T16:02:02.333618Z 2022-01-10T16:02:02.039194Z 2022-01-10T16:02:01.946636Z 2022-01-10T16:02:01.765431Z 2022-01-10T16:02:01.551275Z 2022-01-10T16:02:01.498256Z 2022-01-10T16:02:01.164180Z 2022-01-10T16:02:01.026438Z 2022-01-10T16:02:00.986097Z 2022-01-10T16:02:00.823656Z 2022-01-10T16:02:00.646809Z 2022-01-10T16:02:00.584235Z 2022-01-10T16:02:00.394173Z 2022-01-10T16:02:00.317142Z 2022-01-10T16:02:00.121313Z 2022-01-10T16:01:59.855786Z 2022-01-10T16:01:59.690975Z 2022-01-10T16:01:59.622393Z 2022-01-10T16:01:59.421520Z 2022-01-10T16:01:59.394622Z 2022-01-10T16:01:59.325540Z 2022-01-10T16:01:59.266353Z 2022-01-10T16:01:59.126296Z 2022-01-10T16:01:59.054363Z 2022-01-10T16:01:59.017715Z 2022-01-10T16:01:58.778574Z 2022-01-10T16:01:58.524743Z 2022-01-10T16:01:58.454231Z 2022-01-10T16:01:58.302485Z 2022-01-10T16:01:58.299154Z 2022-01-10T16:01:58.199233Z 2022-01-10T16:01:58.090969Z 2022-01-10T16:01:57.949943Z 2022-01-10T16:01:57.863772Z 2022-01-10T16:01:57.709879Z 2022-01-10T16:01:57.647054Z 2022-01-10T16:01:57.609711Z 2022-01-10T16:01:57.464891Z 2022-01-10T16:01:57.390960Z 2022-01-10T16:01:57.168997Z 2022-01-10T16:01:57.005412Z 2022-01-10T16:01:56.882386Z 2022-01-10T16:01:56.761530Z 2022-01-10T16:01:56.738742Z 2022-01-10T16:01:56.238054Z 2022-01-10T16:01:56.204195Z 2022-01-10T16:01:56.157323Z 2022-01-10T16:01:55.818411Z 2022-01-10T16:01:55.806817Z 2022-01-10T16:01:55.786973Z 2022-01-10T16:01:55.698632Z 2022-01-10T16:01:55.672035Z 2022-01-10T16:01:55.652108Z 2022-01-10T16:01:55.601456Z 2022-01-10T16:01:55.525574Z 2022-01-10T16:01:55.475637Z 2022-01-10T16:01:55.423560Z 2022-01-10T16:01:55.307645Z 2022-01-10T16:01:55.162477Z 2022-01-10T16:01:55.129678Z 2022-01-10T16:01:54.833283Z 2022-01-10T16:01:54.778979Z 2022-01-10T16:01:54.735694Z 2022-01-10T16:01:54.656683Z 2022-01-10T16:01:54.653550Z 2022-01-10T16:01:54.609107Z 2022-01-10T16:01:54.462236Z 2022-01-10T16:01:54.391265Z 2022-01-10T16:01:54.148561Z 2022-01-10T16:01:54.006253Z 2022-01-10T16:01:53.979963Z 2022-01-10T16:01:53.978114Z 2022-01-10T16:01:53.777136Z 2022-01-10T16:01:53.742204Z 2022-01-10T16:01:53.632315Z 2022-01-10T16:01:53.628405Z 2022-01-10T16:01:53.432747Z 2022-01-10T16:01:53.402244Z 2022-01-10T16:01:52.918821Z 2022-01-10T16:01:52.523574Z 2022-01-10T16:01:52.483867Z 2022-01-10T16:06:18.663744Z 2022-01-10T16:06:18.046062Z 2022-01-10T16:06:17.335770Z 2022-01-10T16:06:17.260455Z 2022-01-10T16:06:17.176289Z 2022-01-10T16:06:16.968177Z 2022-01-10T16:06:16.917074Z 2022-01-10T16:06:16.918418Z 2022-01-10T16:06:16.621908Z 2022-01-10T16:06:16.278314Z 2022-01-10T16:06:16.217529Z 2022-01-10T16:06:16.212784Z 2022-01-10T16:06:16.124751Z 2022-01-10T16:06:16.103574Z 2022-01-10T16:06:16.084471Z 2022-01-10T16:06:15.660545Z 2022-01-10T16:06:15.550899Z 2022-01-10T16:06:15.404947Z 2022-01-10T16:06:15.402523Z 2022-01-10T16:06:15.200999Z 2022-01-10T16:06:15.181994Z 2022-01-10T16:06:15.096436Z 2022-01-10T16:06:15.096251Z 2022-01-10T16:06:15.064661Z 2022-01-10T16:06:14.554867Z 2022-01-10T16:06:14.343708Z 2022-01-10T16:06:14.312102Z 2022-01-10T16:06:14.289099Z 2022-01-10T16:06:14.234984Z 2022-01-10T16:06:14.081479Z 2022-01-10T16:06:14.026525Z 2022-01-10T16:06:14.011452Z 2022-01-10T16:06:13.930820Z 2022-01-10T16:06:13.721196Z 2022-01-10T16:06:13.704388Z 2022-01-10T16:06:13.661111Z 2022-01-10T16:06:13.587892Z 2022-01-10T16:06:13.569642Z 2022-01-10T16:06:13.435342Z 2022-01-10T16:06:13.228356Z 2022-01-10T16:06:13.082017Z 2022-01-10T16:06:13.036800Z 2022-01-10T16:06:12.727058Z 2022-01-10T16:06:12.664918Z 2022-01-10T16:06:12.649070Z 2022-01-10T16:06:12.518857Z 2022-01-10T16:06:12.476446Z 2022-01-10T16:06:12.400940Z 2022-01-10T16:06:12.199774Z 2022-01-10T16:06:12.193726Z 2022-01-10T16:06:12.174282Z 2022-01-10T16:06:11.893654Z 2022-01-10T16:06:11.802344Z 2022-01-10T16:06:11.771141Z 2022-01-10T16:06:11.762379Z 2022-01-10T16:06:11.650143Z 2022-01-10T16:06:11.541067Z 2022-01-10T16:06:11.372607Z 2022-01-10T16:06:11.325768Z 2022-01-10T16:06:11.046810Z 2022-01-10T16:06:10.927840Z 2022-01-10T16:06:10.865225Z 2022-01-10T16:06:10.738345Z 2022-01-10T16:06:10.730925Z 2022-01-10T16:06:10.712872Z 2022-01-10T16:06:10.707925Z 2022-01-10T16:06:10.697920Z 2022-01-10T16:06:10.545946Z 2022-01-10T16:06:10.510308Z 2022-01-10T16:06:10.500610Z 2022-01-10T16:06:10.460533Z 2022-01-10T16:06:10.073565Z 2022-01-10T16:06:10.000709Z 2022-01-10T16:06:09.754691Z 2022-01-10T16:06:09.697455Z 2022-01-10T16:06:09.651085Z 2022-01-10T16:06:09.632672Z 2022-01-10T16:06:09.602562Z 2022-01-10T16:06:09.509260Z 2022-01-10T16:06:09.436675Z 2022-01-10T16:06:09.293658Z 2022-01-10T16:06:09.113027Z 2022-01-10T16:06:09.105639Z 2022-01-10T16:06:09.027549Z 2022-01-10T16:06:09.004325Z 2022-01-10T16:06:08.985369Z 2022-01-10T16:06:08.978181Z 2022-01-10T16:06:08.968772Z 2022-01-10T16:06:08.910374Z 2022-01-10T16:06:08.846465Z 2022-01-10T16:06:08.672698Z 2022-01-10T16:06:08.460621Z 2022-01-10T16:06:08.268253Z 2022-01-10T16:06:08.250016Z 2022-01-10T16:06:08.242647Z 2022-01-10T16:06:08.083236Z 2022-01-10T16:06:08.043399Z 2022-01-10T16:06:08.030780Z 2022-01-10T16:06:08.004474Z 2022-01-10T16:06:07.828237Z 2022-01-10T16:10:38.627633Z 2022-01-10T16:10:37.980545Z 2022-01-10T16:10:37.819000Z 2022-01-10T16:10:37.748202Z 2022-01-10T16:10:37.725833Z 2022-01-10T16:10:37.293066Z 2022-01-10T16:10:37.254649Z 2022-01-10T16:10:37.199969Z 2022-01-10T16:10:37.112234Z 2022-01-10T16:10:36.960156Z 2022-01-10T16:10:36.732590Z 2022-01-10T16:10:36.341990Z 2022-01-10T16:10:36.180337Z 2022-01-10T16:10:35.995412Z 2022-01-10T16:10:35.803386Z 2022-01-10T16:10:35.697427Z 2022-01-10T16:10:35.685410Z 2022-01-10T16:10:35.647091Z 2022-01-10T16:10:35.613538Z 2022-01-10T16:10:35.600642Z 2022-01-10T16:10:35.529622Z 2022-01-10T16:10:35.333479Z 2022-01-10T16:10:35.296348Z 2022-01-10T16:10:35.168762Z 2022-01-10T16:10:34.999274Z 2022-01-10T16:10:34.893293Z 2022-01-10T16:10:34.664423Z 2022-01-10T16:10:34.661704Z 2022-01-10T16:10:34.518913Z 2022-01-10T16:10:34.297441Z 2022-01-10T16:10:34.279889Z 2022-01-10T16:10:34.235804Z 2022-01-10T16:10:34.058381Z 2022-01-10T16:10:33.634982Z 2022-01-10T16:10:33.630178Z 2022-01-10T16:10:33.172266Z 2022-01-10T16:10:33.153933Z 2022-01-10T16:10:33.062180Z 2022-01-10T16:10:32.972372Z 2022-01-10T16:10:32.642559Z 2022-01-10T16:10:32.636218Z 2022-01-10T16:10:32.594620Z 2022-01-10T16:10:32.469277Z 2022-01-10T16:10:32.455990Z 2022-01-10T16:10:32.436981Z 2022-01-10T16:10:32.373895Z 2022-01-10T16:10:32.253458Z 2022-01-10T16:10:32.193956Z 2022-01-10T16:10:32.063447Z 2022-01-10T16:10:31.845902Z 2022-01-10T16:10:31.793385Z 2022-01-10T16:10:31.750219Z 2022-01-10T16:10:31.607349Z 2022-01-10T16:10:31.433547Z 2022-01-10T16:10:31.352052Z 2022-01-10T16:10:31.244745Z 2022-01-10T16:10:31.070874Z 2022-01-10T16:10:30.936023Z 2022-01-10T16:10:30.855231Z 2022-01-10T16:10:30.706694Z 2022-01-10T16:10:30.699198Z 2022-01-10T16:10:30.692404Z 2022-01-10T16:10:30.599431Z 2022-01-10T16:10:30.575593Z 2022-01-10T16:10:30.511646Z 2022-01-10T16:10:30.306055Z 2022-01-10T16:10:30.254069Z 2022-01-10T16:10:30.097467Z 2022-01-10T16:10:29.926473Z 2022-01-10T16:10:29.700655Z 2022-01-10T16:10:29.638387Z 2022-01-10T16:10:29.433193Z 2022-01-10T16:10:29.397763Z 2022-01-10T16:10:29.351042Z 2022-01-10T16:10:29.338452Z 2022-01-10T16:10:29.313539Z 2022-01-10T16:10:29.288610Z 2022-01-10T16:10:29.251759Z 2022-01-10T16:10:29.191570Z 2022-01-10T16:10:29.055594Z 2022-01-10T16:10:28.885033Z 2022-01-10T16:10:28.882728Z 2022-01-10T16:10:28.818730Z 2022-01-10T16:10:28.811366Z 2022-01-10T16:10:28.775012Z 2022-01-10T16:10:28.759177Z 2022-01-10T16:10:28.712047Z 2022-01-10T16:10:28.699233Z 2022-01-10T16:10:28.588859Z 2022-01-10T16:10:28.368346Z 2022-01-10T16:10:28.307379Z 2022-01-10T16:10:28.032450Z 2022-01-10T16:10:28.027564Z 2022-01-10T16:10:27.987369Z 2022-01-10T16:10:27.930568Z 2022-01-10T16:10:27.891534Z 2022-01-10T16:10:27.818102Z 2022-01-10T16:10:27.725015Z 2022-01-10T16:10:26.978735Z 2022-01-10T16:10:26.904190Z", + "created_diff": 16.4044666, + "ended": "2022-01-10T16:12:07.602092Z", + "error_job_ids": " []", + "event_first": "2022-01-10T15:52:37.650853Z 2022-01-10T15:53:21.366160Z 2022-01-10T15:53:30.639710Z 2022-01-10T15:53:22.122219Z 2022-01-10T15:54:07.909975Z 2022-01-10T15:53:15.622156Z 2022-01-10T15:54:34.940873Z 2022-01-10T15:53:23.041071Z 2022-01-10T15:53:13.056937Z 2022-01-10T15:53:06.423012Z 2022-01-10T15:55:30.225594Z 2022-01-10T15:53:07.832067Z 2022-01-10T15:55:21.359928Z 2022-01-10T15:53:01.021640Z 2022-01-10T15:54:43.314163Z 2022-01-10T15:53:05.904140Z 2022-01-10T15:54:51.812520Z 2022-01-10T15:53:03.837140Z 2022-01-10T15:54:16.602523Z 2022-01-10T15:52:54.906224Z 2022-01-10T15:53:22.249342Z 2022-01-10T15:52:52.931883Z 2022-01-10T15:53:50.293086Z 2022-01-10T15:52:31.725109Z 2022-01-10T15:53:41.015356Z 2022-01-10T15:52:23.371490Z 2022-01-10T15:53:59.498499Z 2022-01-10T15:52:29.926108Z 2022-01-10T15:55:12.347158Z 2022-01-10T15:52:16.437816Z 2022-01-10T15:55:02.218988Z 2022-01-10T15:52:25.271128Z 2022-01-10T15:54:25.922480Z 2022-01-10T15:52:06.190038Z 2022-01-10T15:52:23.132509Z 2022-01-10T15:52:01.408751Z 2022-01-10T15:53:26.467976Z 2022-01-10T15:53:10.784340Z 2022-01-10T15:53:32.072255Z 2022-01-10T15:53:11.676730Z 2022-01-10T15:53:31.233606Z 2022-01-10T15:53:11.917881Z 2022-01-10T15:53:28.743983Z 2022-01-10T15:53:10.177187Z 2022-01-10T15:53:35.141889Z 2022-01-10T15:53:11.155662Z 2022-01-10T15:53:35.554956Z 2022-01-10T15:53:33.524972Z 2022-01-10T15:53:27.841583Z 2022-01-10T15:53:29.713676Z 2022-01-10T15:53:06.938247Z 2022-01-10T15:53:33.101155Z 2022-01-10T15:53:10.943932Z 2022-01-10T15:53:35.360107Z 2022-01-10T15:53:06.980382Z 2022-01-10T15:53:34.793184Z 2022-01-10T15:53:11.155662Z 2022-01-10T15:53:21.111351Z 2022-01-10T15:53:01.906719Z 2022-01-10T15:53:29.715065Z 2022-01-10T15:53:08.114740Z 2022-01-10T15:53:24.659867Z 2022-01-10T15:53:06.938247Z 2022-01-10T15:53:27.544016Z 2022-01-10T15:53:05.341718Z 2022-01-10T15:53:25.447117Z 2022-01-10T15:53:03.831165Z 2022-01-10T15:53:35.560072Z 2022-01-10T15:52:59.874866Z 2022-01-10T15:53:25.447117Z 2022-01-10T15:53:03.570753Z 2022-01-10T15:53:34.020954Z 2022-01-10T15:53:03.065067Z 2022-01-10T15:53:00.889625Z 2022-01-10T15:53:31.233606Z 2022-01-10T15:53:21.872400Z 2022-01-10T15:52:59.352699Z 2022-01-10T15:53:28.050252Z 2022-01-10T15:52:59.355042Z 2022-01-10T15:53:23.483943Z 2022-01-10T15:52:53.848983Z 2022-01-10T15:53:33.623004Z 2022-01-10T15:52:54.637347Z 2022-01-10T15:53:20.858804Z 2022-01-10T15:52:52.579629Z 2022-01-10T15:53:18.714956Z 2022-01-10T15:52:52.579716Z 2022-01-10T15:53:22.370419Z 2022-01-10T15:52:52.370816Z 2022-01-10T15:53:25.693203Z 2022-01-10T15:52:53.376047Z 2022-01-10T15:53:17.455968Z 2022-01-10T15:52:49.751711Z 2022-01-10T15:53:20.003057Z 2022-01-10T15:52:47.879655Z 2022-01-10T15:53:16.906868Z 2022-01-10T15:52:44.657435Z 2022-01-10T15:53:25.051874Z 2022-01-10T15:52:51.942405Z 2022-01-10T15:53:17.644874Z 2022-01-10T15:59:24.158569Z 2022-01-10T15:59:23.053623Z 2022-01-10T15:59:23.639632Z 2022-01-10T15:59:21.917988Z 2022-01-10T15:59:23.100896Z 2022-01-10T15:59:20.376889Z 2022-01-10T15:59:21.032744Z 2022-01-10T15:59:18.437367Z 2022-01-10T15:59:19.236950Z 2022-01-10T15:59:19.730701Z 2022-01-10T15:59:18.582993Z 2022-01-10T15:59:14.752176Z 2022-01-10T15:59:19.341182Z 2022-01-10T15:59:19.616257Z 2022-01-10T15:59:16.704649Z 2022-01-10T15:59:16.329999Z 2022-01-10T15:59:16.966556Z 2022-01-10T15:59:16.561188Z 2022-01-10T15:59:18.143793Z 2022-01-10T15:59:16.329545Z 2022-01-10T15:59:15.586279Z 2022-01-10T15:59:13.740306Z 2022-01-10T15:59:13.588691Z 2022-01-10T15:59:15.011088Z 2022-01-10T15:59:17.249684Z 2022-01-10T15:59:14.752176Z 2022-01-10T15:59:13.477743Z 2022-01-10T15:59:15.011088Z 2022-01-10T15:59:17.504668Z 2022-01-10T15:59:11.355986Z 2022-01-10T15:59:12.714676Z 2022-01-10T15:59:08.698977Z 2022-01-10T15:59:14.998659Z 2022-01-10T15:59:05.524092Z 2022-01-10T15:59:12.208004Z 2022-01-10T15:59:07.094706Z 2022-01-10T15:59:06.915276Z 2022-01-10T15:59:11.822026Z 2022-01-10T15:59:11.646700Z 2022-01-10T15:59:13.499088Z 2022-01-10T15:59:12.975016Z 2022-01-10T15:59:14.495968Z 2022-01-10T15:59:10.297100Z 2022-01-10T15:59:09.208675Z 2022-01-10T15:59:02.126080Z 2022-01-10T15:59:03.498962Z 2022-01-10T15:59:06.477741Z 2022-01-10T15:59:04.427888Z 2022-01-10T15:59:06.049576Z 2022-01-10T15:59:08.907105Z 2022-01-10T15:59:04.019727Z 2022-01-10T15:59:07.887069Z 2022-01-10T15:59:06.583172Z 2022-01-10T15:59:09.923375Z 2022-01-10T15:58:59.884817Z 2022-01-10T15:59:01.889454Z 2022-01-10T15:59:00.037682Z 2022-01-10T15:59:07.591441Z 2022-01-10T15:59:00.448105Z 2022-01-10T15:58:56.840236Z 2022-01-10T15:58:53.839637Z 2022-01-10T15:58:50.651432Z 2022-01-10T15:59:02.821714Z 2022-01-10T15:59:05.814701Z 2022-01-10T15:58:48.124465Z 2022-01-10T15:58:53.305860Z 2022-01-10T15:58:53.666761Z 2022-01-10T15:58:50.967693Z 2022-01-10T15:58:54.606784Z 2022-01-10T15:58:56.048945Z 2022-01-10T15:58:57.756093Z 2022-01-10T15:58:45.117712Z 2022-01-10T15:58:45.348263Z 2022-01-10T15:58:46.158305Z 2022-01-10T15:58:42.625525Z 2022-01-10T15:58:46.099002Z 2022-01-10T15:58:37.192520Z 2022-01-10T15:58:18.334132Z 2022-01-10T15:58:12.351712Z 2022-01-10T15:58:20.116364Z 2022-01-10T15:58:12.856656Z 2022-01-10T15:58:15.933740Z 2022-01-10T15:58:10.595184Z 2022-01-10T15:58:11.963049Z 2022-01-10T15:58:09.841680Z 2022-01-10T15:58:12.470861Z 2022-01-10T15:58:07.208408Z 2022-01-10T15:58:10.592283Z 2022-01-10T15:58:08.244905Z 2022-01-10T15:58:08.113444Z 2022-01-10T15:58:04.477673Z 2022-01-10T15:58:09.054938Z 2022-01-10T15:57:59.085922Z 2022-01-10T15:57:57.993776Z 2022-01-10T15:57:56.906996Z 2022-01-10T15:58:00.746625Z 2022-01-10T15:57:54.650144Z 2022-01-10T15:57:55.194077Z 2022-01-10T15:57:52.466449Z 2022-01-10T15:57:52.351794Z 2022-01-10T16:03:32.802143Z 2022-01-10T16:03:30.447696Z 2022-01-10T16:03:32.604738Z 2022-01-10T16:03:28.721579Z 2022-01-10T16:03:33.313654Z 2022-01-10T16:03:23.614912Z 2022-01-10T16:03:27.603120Z 2022-01-10T16:03:27.543790Z 2022-01-10T16:03:30.452679Z 2022-01-10T16:03:30.538356Z 2022-01-10T16:03:26.622364Z 2022-01-10T16:03:22.925803Z 2022-01-10T16:03:25.362678Z 2022-01-10T16:03:26.749261Z 2022-01-10T16:03:27.100548Z 2022-01-10T16:03:27.543790Z 2022-01-10T16:03:27.672656Z 2022-01-10T16:03:27.623252Z 2022-01-10T16:03:25.863638Z 2022-01-10T16:03:27.543790Z 2022-01-10T16:03:27.189887Z 2022-01-10T16:03:24.980090Z 2022-01-10T16:03:25.058741Z 2022-01-10T16:03:23.317141Z 2022-01-10T16:03:21.481627Z 2022-01-10T16:03:20.980910Z 2022-01-10T16:03:25.862486Z 2022-01-10T16:03:26.177977Z 2022-01-10T16:03:27.357740Z 2022-01-10T16:03:28.241880Z 2022-01-10T16:03:20.730772Z 2022-01-10T16:03:22.399904Z 2022-01-10T16:03:22.823904Z 2022-01-10T16:03:13.340378Z 2022-01-10T16:03:20.216984Z 2022-01-10T16:03:29.529666Z 2022-01-10T16:03:17.209725Z 2022-01-10T16:03:19.844997Z 2022-01-10T16:03:19.459703Z 2022-01-10T16:03:24.102539Z 2022-01-10T16:03:21.483707Z 2022-01-10T16:03:16.457678Z 2022-01-10T16:03:18.450633Z 2022-01-10T16:03:26.677588Z 2022-01-10T16:03:12.098635Z 2022-01-10T16:03:14.861978Z 2022-01-10T16:03:19.459703Z 2022-01-10T16:03:13.182991Z 2022-01-10T16:03:15.687478Z 2022-01-10T16:03:11.698333Z 2022-01-10T16:03:10.678768Z 2022-01-10T16:03:13.046988Z 2022-01-10T16:03:14.180685Z 2022-01-10T16:03:13.340378Z 2022-01-10T16:03:19.204717Z 2022-01-10T16:03:20.731360Z 2022-01-10T16:03:14.431597Z 2022-01-10T16:03:15.448105Z 2022-01-10T16:03:13.450787Z 2022-01-10T16:03:06.335813Z 2022-01-10T16:03:13.939794Z 2022-01-10T16:02:59.587000Z 2022-01-10T16:03:05.295987Z 2022-01-10T16:03:09.168351Z 2022-01-10T16:03:01.775937Z 2022-01-10T16:03:03.909958Z 2022-01-10T16:03:03.241750Z 2022-01-10T16:03:05.819972Z 2022-01-10T16:03:05.041335Z 2022-01-10T16:03:03.909871Z 2022-01-10T16:03:01.418475Z 2022-01-10T16:02:56.575267Z 2022-01-10T16:02:55.903837Z 2022-01-10T16:02:41.241076Z 2022-01-10T16:02:57.633838Z 2022-01-10T16:02:41.563389Z 2022-01-10T16:02:53.486448Z 2022-01-10T16:02:58.593428Z 2022-01-10T16:02:50.995692Z 2022-01-10T16:02:22.173918Z 2022-01-10T16:02:16.084970Z 2022-01-10T16:02:36.112933Z 2022-01-10T16:02:18.992005Z 2022-01-10T16:02:16.084970Z 2022-01-10T16:02:20.192834Z 2022-01-10T16:02:16.348965Z 2022-01-10T16:02:19.728867Z 2022-01-10T16:02:15.584567Z 2022-01-10T16:02:16.725578Z 2022-01-10T16:02:16.697925Z 2022-01-10T16:02:15.298719Z 2022-01-10T16:02:14.213744Z 2022-01-10T16:02:14.470379Z 2022-01-10T16:02:14.895040Z 2022-01-10T16:02:07.289395Z 2022-01-10T16:02:09.236101Z 2022-01-10T16:02:05.197969Z 2022-01-10T16:02:05.491015Z 2022-01-10T16:02:01.730487Z 2022-01-10T16:02:01.027861Z 2022-01-10T16:07:44.633116Z 2022-01-10T16:07:44.899138Z 2022-01-10T16:07:42.928894Z 2022-01-10T16:07:44.973302Z 2022-01-10T16:07:41.928629Z 2022-01-10T16:07:47.056076Z 2022-01-10T16:07:44.273959Z 2022-01-10T16:07:43.615184Z 2022-01-10T16:07:41.411366Z 2022-01-10T16:07:43.827757Z 2022-01-10T16:07:41.678066Z 2022-01-10T16:07:44.273959Z 2022-01-10T16:07:41.162046Z 2022-01-10T16:07:45.304908Z 2022-01-10T16:07:40.427615Z 2022-01-10T16:07:44.589702Z 2022-01-10T16:07:39.286854Z 2022-01-10T16:07:42.450678Z 2022-01-10T16:07:42.928894Z 2022-01-10T16:07:45.726845Z 2022-01-10T16:07:36.281880Z 2022-01-10T16:07:44.275380Z 2022-01-10T16:07:39.003947Z 2022-01-10T16:07:39.422695Z 2022-01-10T16:07:39.559706Z 2022-01-10T16:07:38.068603Z 2022-01-10T16:07:34.817809Z 2022-01-10T16:07:37.399882Z 2022-01-10T16:07:37.112666Z 2022-01-10T16:07:35.132173Z 2022-01-10T16:07:36.825876Z 2022-01-10T16:07:44.052001Z 2022-01-10T16:07:34.595769Z 2022-01-10T16:07:35.939794Z 2022-01-10T16:07:39.248941Z 2022-01-10T16:07:40.170677Z 2022-01-10T16:07:34.313010Z 2022-01-10T16:07:31.741812Z 2022-01-10T16:07:37.112837Z 2022-01-10T16:07:41.845711Z 2022-01-10T16:07:30.543433Z 2022-01-10T16:07:39.422695Z 2022-01-10T16:07:25.951064Z 2022-01-10T16:07:35.258464Z 2022-01-10T16:07:35.479392Z 2022-01-10T16:07:33.916065Z 2022-01-10T16:07:35.029987Z 2022-01-10T16:07:26.452804Z 2022-01-10T16:07:27.765944Z 2022-01-10T16:07:41.439584Z 2022-01-10T16:07:33.569895Z 2022-01-10T16:07:37.399882Z 2022-01-10T16:07:25.494802Z 2022-01-10T16:07:29.300856Z 2022-01-10T16:07:24.630007Z 2022-01-10T16:07:25.998554Z 2022-01-10T16:07:28.287823Z 2022-01-10T16:07:30.155713Z 2022-01-10T16:07:24.278180Z 2022-01-10T16:07:35.798798Z 2022-01-10T16:07:25.749007Z 2022-01-10T16:07:28.349600Z 2022-01-10T16:07:22.520933Z 2022-01-10T16:07:25.644844Z 2022-01-10T16:07:18.969911Z 2022-01-10T16:07:25.970722Z 2022-01-10T16:07:14.449852Z 2022-01-10T16:07:24.971254Z 2022-01-10T16:07:20.083374Z 2022-01-10T16:07:16.561367Z 2022-01-10T16:07:11.086577Z 2022-01-10T16:07:27.213954Z 2022-01-10T16:07:02.273163Z 2022-01-10T16:06:48.130567Z 2022-01-10T16:06:54.117173Z 2022-01-10T16:06:42.939877Z 2022-01-10T16:06:45.705243Z 2022-01-10T16:06:51.712309Z 2022-01-10T16:06:59.820709Z 2022-01-10T16:06:42.215346Z 2022-01-10T16:06:50.541978Z 2022-01-10T16:06:39.697937Z 2022-01-10T16:06:43.663979Z 2022-01-10T16:06:47.734060Z 2022-01-10T16:06:42.628790Z 2022-01-10T16:06:37.022893Z 2022-01-10T16:06:37.527882Z 2022-01-10T16:06:37.655158Z 2022-01-10T16:06:32.881855Z 2022-01-10T16:06:35.231800Z 2022-01-10T16:06:33.403326Z 2022-01-10T16:06:31.844931Z 2022-01-10T16:06:35.488684Z 2022-01-10T16:06:30.797910Z 2022-01-10T16:06:31.380611Z 2022-01-10T16:06:22.256911Z 2022-01-10T16:06:20.789933Z 2022-01-10T16:06:21.504979Z 2022-01-10T16:06:18.832062Z 2022-01-10T16:06:19.842295Z 2022-01-10T16:12:05.486927Z 2022-01-10T16:12:00.890593Z 2022-01-10T16:12:04.966957Z 2022-01-10T16:12:01.863539Z 2022-01-10T16:12:03.293736Z 2022-01-10T16:11:54.727929Z 2022-01-10T16:12:01.933072Z 2022-01-10T16:11:59.941117Z 2022-01-10T16:12:00.426006Z 2022-01-10T16:11:58.894545Z 2022-01-10T16:12:02.836177Z 2022-01-10T16:12:00.137856Z 2022-01-10T16:12:03.356800Z 2022-01-10T16:11:57.990753Z 2022-01-10T16:12:01.205867Z 2022-01-10T16:11:57.634541Z 2022-01-10T16:12:04.019238Z 2022-01-10T16:11:57.336681Z 2022-01-10T16:11:57.892782Z 2022-01-10T16:12:01.244604Z 2022-01-10T16:11:59.652987Z 2022-01-10T16:12:00.145567Z 2022-01-10T16:12:00.680252Z 2022-01-10T16:11:57.067834Z 2022-01-10T16:11:58.131306Z 2022-01-10T16:11:57.336681Z 2022-01-10T16:11:48.192080Z 2022-01-10T16:11:49.715768Z 2022-01-10T16:11:52.539118Z 2022-01-10T16:11:54.547563Z 2022-01-10T16:11:59.911241Z 2022-01-10T16:11:55.489887Z 2022-01-10T16:11:56.131822Z 2022-01-10T16:11:49.472276Z 2022-01-10T16:11:55.119665Z 2022-01-10T16:11:47.376075Z 2022-01-10T16:11:52.814976Z 2022-01-10T16:11:44.831453Z 2022-01-10T16:11:45.452981Z 2022-01-10T16:11:41.566644Z 2022-01-10T16:11:50.233127Z 2022-01-10T16:11:43.377855Z 2022-01-10T16:11:23.305816Z 2022-01-10T16:11:43.417297Z 2022-01-10T16:11:39.281677Z 2022-01-10T16:11:49.572777Z 2022-01-10T16:11:20.103021Z 2022-01-10T16:11:44.051050Z 2022-01-10T16:11:48.829792Z 2022-01-10T16:11:21.781702Z 2022-01-10T16:11:20.338468Z 2022-01-10T16:11:22.539357Z 2022-01-10T16:11:21.738658Z 2022-01-10T16:11:19.755006Z 2022-01-10T16:11:19.828584Z 2022-01-10T16:11:21.783897Z 2022-01-10T16:11:19.828584Z 2022-01-10T16:11:17.894060Z 2022-01-10T16:11:17.855448Z 2022-01-10T16:11:22.043396Z 2022-01-10T16:11:18.863874Z 2022-01-10T16:11:20.527209Z 2022-01-10T16:11:16.279075Z 2022-01-10T16:11:14.307566Z 2022-01-10T16:11:18.535806Z 2022-01-10T16:11:15.531322Z 2022-01-10T16:11:17.115535Z 2022-01-10T16:11:15.783012Z 2022-01-10T16:11:10.428620Z 2022-01-10T16:11:15.573978Z 2022-01-10T16:11:14.204419Z 2022-01-10T16:11:14.798362Z 2022-01-10T16:11:10.282707Z 2022-01-10T16:11:12.309980Z 2022-01-10T16:11:10.892621Z 2022-01-10T16:11:09.255475Z 2022-01-10T16:11:15.521038Z 2022-01-10T16:11:08.872095Z 2022-01-10T16:11:09.713638Z 2022-01-10T16:11:13.598928Z 2022-01-10T16:11:09.578734Z 2022-01-10T16:11:10.192849Z 2022-01-10T16:11:07.038514Z 2022-01-10T16:11:02.213541Z 2022-01-10T16:11:08.027934Z 2022-01-10T16:11:02.782325Z 2022-01-10T16:11:06.864570Z 2022-01-10T16:11:04.382003Z 2022-01-10T16:11:09.419743Z 2022-01-10T16:11:00.963885Z 2022-01-10T16:11:01.645539Z 2022-01-10T16:10:44.830355Z 2022-01-10T16:10:42.757515Z 2022-01-10T16:10:43.513922Z 2022-01-10T16:10:45.389690Z 2022-01-10T16:10:44.521341Z 2022-01-10T16:10:42.077051Z 2022-01-10T16:10:40.655142Z 2022-01-10T16:10:33.758635Z 2022-01-10T16:10:35.345063Z", + "event_last": "2022-01-10T15:52:40.059470Z 2022-01-10T15:53:24.957924Z 2022-01-10T15:53:31.064523Z 2022-01-10T15:53:27.926747Z 2022-01-10T15:54:08.349455Z 2022-01-10T15:53:20.285810Z 2022-01-10T15:54:35.314446Z 2022-01-10T15:53:28.081078Z 2022-01-10T15:53:14.130874Z 2022-01-10T15:53:11.953803Z 2022-01-10T15:55:30.643595Z 2022-01-10T15:53:12.749760Z 2022-01-10T15:55:21.916866Z 2022-01-10T15:53:05.949799Z 2022-01-10T15:54:43.692166Z 2022-01-10T15:53:09.853458Z 2022-01-10T15:54:52.657849Z 2022-01-10T15:53:08.659092Z 2022-01-10T15:54:17.025749Z 2022-01-10T15:52:59.884698Z 2022-01-10T15:53:22.464923Z 2022-01-10T15:52:57.775187Z 2022-01-10T15:53:51.428434Z 2022-01-10T15:52:38.035725Z 2022-01-10T15:53:41.200417Z 2022-01-10T15:52:27.633966Z 2022-01-10T15:53:59.894085Z 2022-01-10T15:52:35.283331Z 2022-01-10T15:55:12.491399Z 2022-01-10T15:52:19.609733Z 2022-01-10T15:55:02.668778Z 2022-01-10T15:52:30.518878Z 2022-01-10T15:54:26.376461Z 2022-01-10T15:52:07.514130Z 2022-01-10T15:52:26.956433Z 2022-01-10T15:52:02.995596Z 2022-01-10T15:53:32.203392Z 2022-01-10T15:53:12.879566Z 2022-01-10T15:53:36.295193Z 2022-01-10T15:53:13.434790Z 2022-01-10T15:53:34.964273Z 2022-01-10T15:53:13.571838Z 2022-01-10T15:53:32.729699Z 2022-01-10T15:53:11.722087Z 2022-01-10T15:53:37.533739Z 2022-01-10T15:53:13.278680Z 2022-01-10T15:53:37.956193Z 2022-01-10T15:53:37.308348Z 2022-01-10T15:53:31.789070Z 2022-01-10T15:53:34.494611Z 2022-01-10T15:53:08.679551Z 2022-01-10T15:53:37.225340Z 2022-01-10T15:53:12.639398Z 2022-01-10T15:53:37.956193Z 2022-01-10T15:53:09.333160Z 2022-01-10T15:53:38.216164Z 2022-01-10T15:53:13.409361Z 2022-01-10T15:53:24.906701Z 2022-01-10T15:53:05.179391Z 2022-01-10T15:53:36.121539Z 2022-01-10T15:53:10.076902Z 2022-01-10T15:53:30.082984Z 2022-01-10T15:53:09.645493Z 2022-01-10T15:53:32.295654Z 2022-01-10T15:53:07.947563Z 2022-01-10T15:53:30.958758Z 2022-01-10T15:53:06.860449Z 2022-01-10T15:53:38.117350Z 2022-01-10T15:53:03.708486Z 2022-01-10T15:53:32.392269Z 2022-01-10T15:53:06.485427Z 2022-01-10T15:53:37.763048Z 2022-01-10T15:53:06.908441Z 2022-01-10T15:53:03.463048Z 2022-01-10T15:53:34.393138Z 2022-01-10T15:53:25.698903Z 2022-01-10T15:53:01.415971Z 2022-01-10T15:53:32.728291Z 2022-01-10T15:53:02.386244Z 2022-01-10T15:53:29.848840Z 2022-01-10T15:52:56.914181Z 2022-01-10T15:53:37.481860Z 2022-01-10T15:52:57.706932Z 2022-01-10T15:53:26.562398Z 2022-01-10T15:52:56.577496Z 2022-01-10T15:53:23.155764Z 2022-01-10T15:52:56.137464Z 2022-01-10T15:53:27.416891Z 2022-01-10T15:52:57.208459Z 2022-01-10T15:53:28.553949Z 2022-01-10T15:52:56.397518Z 2022-01-10T15:53:21.432926Z 2022-01-10T15:52:52.620821Z 2022-01-10T15:53:25.437489Z 2022-01-10T15:52:50.721417Z 2022-01-10T15:53:21.516791Z 2022-01-10T15:52:48.543897Z 2022-01-10T15:53:30.547743Z 2022-01-10T15:52:55.235396Z 2022-01-10T15:53:21.965290Z 2022-01-10T15:59:26.084753Z 2022-01-10T15:59:23.534234Z 2022-01-10T15:59:24.148651Z 2022-01-10T15:59:22.657924Z 2022-01-10T15:59:23.740003Z 2022-01-10T15:59:21.792062Z 2022-01-10T15:59:23.105006Z 2022-01-10T15:59:20.702798Z 2022-01-10T15:59:21.329026Z 2022-01-10T15:59:21.508985Z 2022-01-10T15:59:20.894443Z 2022-01-10T15:59:17.548851Z 2022-01-10T15:59:21.458472Z 2022-01-10T15:59:21.548400Z 2022-01-10T15:59:18.815851Z 2022-01-10T15:59:19.516886Z 2022-01-10T15:59:20.489519Z 2022-01-10T15:59:20.165155Z 2022-01-10T15:59:20.708111Z 2022-01-10T15:59:18.805708Z 2022-01-10T15:59:19.165750Z 2022-01-10T15:59:16.719690Z 2022-01-10T15:59:16.820844Z 2022-01-10T15:59:17.810683Z 2022-01-10T15:59:20.349849Z 2022-01-10T15:59:18.311787Z 2022-01-10T15:59:16.678510Z 2022-01-10T15:59:17.699556Z 2022-01-10T15:59:20.067617Z 2022-01-10T15:59:14.397803Z 2022-01-10T15:59:16.117100Z 2022-01-10T15:59:12.597449Z 2022-01-10T15:59:18.386390Z 2022-01-10T15:59:08.757312Z 2022-01-10T15:59:15.824116Z 2022-01-10T15:59:11.187368Z 2022-01-10T15:59:10.409969Z 2022-01-10T15:59:14.835696Z 2022-01-10T15:59:15.118196Z 2022-01-10T15:59:16.824916Z 2022-01-10T15:59:15.629335Z 2022-01-10T15:59:17.893716Z 2022-01-10T15:59:12.711664Z 2022-01-10T15:59:12.439554Z 2022-01-10T15:59:06.410465Z 2022-01-10T15:59:06.544012Z 2022-01-10T15:59:11.142647Z 2022-01-10T15:59:07.683808Z 2022-01-10T15:59:10.957049Z 2022-01-10T15:59:12.525720Z 2022-01-10T15:59:09.121470Z 2022-01-10T15:59:11.247700Z 2022-01-10T15:59:09.444823Z 2022-01-10T15:59:13.381718Z 2022-01-10T15:59:03.799454Z 2022-01-10T15:59:05.277768Z 2022-01-10T15:59:02.598436Z 2022-01-10T15:59:10.440706Z 2022-01-10T15:59:03.796518Z 2022-01-10T15:58:59.848945Z 2022-01-10T15:58:57.628453Z 2022-01-10T15:58:53.442844Z 2022-01-10T15:59:07.242425Z 2022-01-10T15:59:09.229136Z 2022-01-10T15:58:52.785021Z 2022-01-10T15:58:56.888244Z 2022-01-10T15:58:57.470192Z 2022-01-10T15:58:54.705177Z 2022-01-10T15:58:58.270654Z 2022-01-10T15:58:59.104906Z 2022-01-10T15:59:02.499836Z 2022-01-10T15:58:48.399187Z 2022-01-10T15:58:48.649886Z 2022-01-10T15:58:49.794766Z 2022-01-10T15:58:46.353942Z 2022-01-10T15:58:50.967693Z 2022-01-10T15:58:41.672266Z 2022-01-10T15:58:21.095486Z 2022-01-10T15:58:13.536533Z 2022-01-10T15:58:23.209031Z 2022-01-10T15:58:14.343697Z 2022-01-10T15:58:18.822671Z 2022-01-10T15:58:11.754649Z 2022-01-10T15:58:13.528786Z 2022-01-10T15:58:10.961315Z 2022-01-10T15:58:14.367848Z 2022-01-10T15:58:08.499420Z 2022-01-10T15:58:12.138162Z 2022-01-10T15:58:09.252449Z 2022-01-10T15:58:09.210742Z 2022-01-10T15:58:05.764196Z 2022-01-10T15:58:10.334230Z 2022-01-10T15:58:00.087498Z 2022-01-10T15:58:00.746625Z 2022-01-10T15:57:57.763353Z 2022-01-10T15:58:01.865331Z 2022-01-10T15:57:55.499195Z 2022-01-10T15:57:56.398023Z 2022-01-10T15:57:54.661688Z 2022-01-10T15:57:53.875660Z 2022-01-10T16:03:33.313654Z 2022-01-10T16:03:31.659274Z 2022-01-10T16:03:33.337347Z 2022-01-10T16:03:30.922169Z 2022-01-10T16:03:35.689329Z 2022-01-10T16:03:26.218120Z 2022-01-10T16:03:30.113560Z 2022-01-10T16:03:30.197756Z 2022-01-10T16:03:31.659274Z 2022-01-10T16:03:32.334774Z 2022-01-10T16:03:29.059302Z 2022-01-10T16:03:25.804699Z 2022-01-10T16:03:26.994502Z 2022-01-10T16:03:30.118749Z 2022-01-10T16:03:29.643034Z 2022-01-10T16:03:30.118749Z 2022-01-10T16:03:30.578502Z 2022-01-10T16:03:30.031788Z 2022-01-10T16:03:28.242915Z 2022-01-10T16:03:29.538444Z 2022-01-10T16:03:30.371434Z 2022-01-10T16:03:27.891742Z 2022-01-10T16:03:27.233047Z 2022-01-10T16:03:26.773816Z 2022-01-10T16:03:25.421737Z 2022-01-10T16:03:24.091235Z 2022-01-10T16:03:28.721579Z 2022-01-10T16:03:29.134082Z 2022-01-10T16:03:29.802331Z 2022-01-10T16:03:30.960866Z 2022-01-10T16:03:23.874966Z 2022-01-10T16:03:26.490948Z 2022-01-10T16:03:25.573443Z 2022-01-10T16:03:17.348454Z 2022-01-10T16:03:23.625557Z 2022-01-10T16:03:31.920981Z 2022-01-10T16:03:20.807328Z 2022-01-10T16:03:22.369358Z 2022-01-10T16:03:22.192481Z 2022-01-10T16:03:27.591750Z 2022-01-10T16:03:24.882918Z 2022-01-10T16:03:19.740444Z 2022-01-10T16:03:22.134148Z 2022-01-10T16:03:29.525932Z 2022-01-10T16:03:16.036744Z 2022-01-10T16:03:19.060746Z 2022-01-10T16:03:23.833565Z 2022-01-10T16:03:19.303764Z 2022-01-10T16:03:19.566453Z 2022-01-10T16:03:16.582170Z 2022-01-10T16:03:15.028846Z 2022-01-10T16:03:16.597952Z 2022-01-10T16:03:17.387448Z 2022-01-10T16:03:17.610479Z 2022-01-10T16:03:22.671823Z 2022-01-10T16:03:23.504339Z 2022-01-10T16:03:17.852526Z 2022-01-10T16:03:20.114147Z 2022-01-10T16:03:17.038731Z 2022-01-10T16:03:10.218972Z 2022-01-10T16:03:17.830469Z 2022-01-10T16:03:03.642387Z 2022-01-10T16:03:10.732923Z 2022-01-10T16:03:13.274092Z 2022-01-10T16:03:05.871057Z 2022-01-10T16:03:07.538683Z 2022-01-10T16:03:07.650268Z 2022-01-10T16:03:09.412729Z 2022-01-10T16:03:09.225591Z 2022-01-10T16:03:08.392798Z 2022-01-10T16:03:05.576522Z 2022-01-10T16:02:59.890539Z 2022-01-10T16:02:59.667764Z 2022-01-10T16:02:44.753715Z 2022-01-10T16:03:01.995416Z 2022-01-10T16:02:45.774709Z 2022-01-10T16:02:57.185491Z 2022-01-10T16:03:03.029462Z 2022-01-10T16:02:54.025462Z 2022-01-10T16:02:24.996394Z 2022-01-10T16:02:17.264815Z 2022-01-10T16:02:40.987616Z 2022-01-10T16:02:20.916241Z 2022-01-10T16:02:17.906024Z 2022-01-10T16:02:23.000566Z 2022-01-10T16:02:17.481881Z 2022-01-10T16:02:21.550888Z 2022-01-10T16:02:16.683125Z 2022-01-10T16:02:17.852949Z 2022-01-10T16:02:18.519175Z 2022-01-10T16:02:16.632595Z 2022-01-10T16:02:15.287452Z 2022-01-10T16:02:15.435478Z 2022-01-10T16:02:16.360355Z 2022-01-10T16:02:08.854392Z 2022-01-10T16:02:10.266727Z 2022-01-10T16:02:06.711804Z 2022-01-10T16:02:06.670966Z 2022-01-10T16:02:02.977547Z 2022-01-10T16:02:02.361773Z 2022-01-10T16:07:45.925549Z 2022-01-10T16:07:47.619345Z 2022-01-10T16:07:45.140701Z 2022-01-10T16:07:47.338406Z 2022-01-10T16:07:44.290712Z 2022-01-10T16:07:49.074648Z 2022-01-10T16:07:47.111704Z 2022-01-10T16:07:45.545696Z 2022-01-10T16:07:44.101698Z 2022-01-10T16:07:46.120667Z 2022-01-10T16:07:43.919706Z 2022-01-10T16:07:46.864493Z 2022-01-10T16:07:43.087763Z 2022-01-10T16:07:48.113278Z 2022-01-10T16:07:42.808722Z 2022-01-10T16:07:47.539359Z 2022-01-10T16:07:42.294142Z 2022-01-10T16:07:44.899138Z 2022-01-10T16:07:44.817923Z 2022-01-10T16:07:48.597800Z 2022-01-10T16:07:39.550708Z 2022-01-10T16:07:46.972906Z 2022-01-10T16:07:42.808722Z 2022-01-10T16:07:42.238558Z 2022-01-10T16:07:42.574875Z 2022-01-10T16:07:42.083590Z 2022-01-10T16:07:38.802168Z 2022-01-10T16:07:41.746311Z 2022-01-10T16:07:40.790284Z 2022-01-10T16:07:38.978394Z 2022-01-10T16:07:39.740305Z 2022-01-10T16:07:46.688046Z 2022-01-10T16:07:38.876261Z 2022-01-10T16:07:39.313511Z 2022-01-10T16:07:42.720012Z 2022-01-10T16:07:43.524734Z 2022-01-10T16:07:37.634744Z 2022-01-10T16:07:35.568505Z 2022-01-10T16:07:40.972735Z 2022-01-10T16:07:45.830455Z 2022-01-10T16:07:33.449317Z 2022-01-10T16:07:43.103548Z 2022-01-10T16:07:30.020719Z 2022-01-10T16:07:39.745709Z 2022-01-10T16:07:38.877488Z 2022-01-10T16:07:37.935432Z 2022-01-10T16:07:38.713890Z 2022-01-10T16:07:30.800413Z 2022-01-10T16:07:31.117888Z 2022-01-10T16:07:44.335505Z 2022-01-10T16:07:36.706001Z 2022-01-10T16:07:41.541435Z 2022-01-10T16:07:29.542926Z 2022-01-10T16:07:33.292743Z 2022-01-10T16:07:28.059872Z 2022-01-10T16:07:29.667503Z 2022-01-10T16:07:32.137143Z 2022-01-10T16:07:33.055431Z 2022-01-10T16:07:27.255172Z 2022-01-10T16:07:40.486521Z 2022-01-10T16:07:30.374908Z 2022-01-10T16:07:32.819442Z 2022-01-10T16:07:25.487716Z 2022-01-10T16:07:30.913470Z 2022-01-10T16:07:22.828718Z 2022-01-10T16:07:29.595900Z 2022-01-10T16:07:17.456717Z 2022-01-10T16:07:28.617647Z 2022-01-10T16:07:23.836868Z 2022-01-10T16:07:20.320577Z 2022-01-10T16:07:14.422348Z 2022-01-10T16:07:30.482445Z 2022-01-10T16:07:07.367424Z 2022-01-10T16:06:52.689388Z 2022-01-10T16:06:58.917489Z 2022-01-10T16:06:45.918545Z 2022-01-10T16:06:49.817476Z 2022-01-10T16:06:55.202182Z 2022-01-10T16:07:03.042396Z 2022-01-10T16:06:45.737671Z 2022-01-10T16:06:55.602628Z 2022-01-10T16:06:42.272768Z 2022-01-10T16:06:47.148897Z 2022-01-10T16:06:51.582257Z 2022-01-10T16:06:45.935423Z 2022-01-10T16:06:39.795755Z 2022-01-10T16:06:40.240473Z 2022-01-10T16:06:40.544139Z 2022-01-10T16:06:35.429531Z 2022-01-10T16:06:37.618063Z 2022-01-10T16:06:35.823135Z 2022-01-10T16:06:32.936807Z 2022-01-10T16:06:38.102039Z 2022-01-10T16:06:31.808690Z 2022-01-10T16:06:33.170785Z 2022-01-10T16:06:25.129786Z 2022-01-10T16:06:23.944910Z 2022-01-10T16:06:23.026952Z 2022-01-10T16:06:20.798270Z 2022-01-10T16:06:20.746757Z 2022-01-10T16:12:07.602092Z 2022-01-10T16:12:01.893993Z 2022-01-10T16:12:05.752793Z 2022-01-10T16:12:02.625568Z 2022-01-10T16:12:04.832744Z 2022-01-10T16:11:56.271091Z 2022-01-10T16:12:03.923541Z 2022-01-10T16:12:01.749317Z 2022-01-10T16:12:02.273969Z 2022-01-10T16:12:00.215257Z 2022-01-10T16:12:04.120130Z 2022-01-10T16:12:01.579761Z 2022-01-10T16:12:04.536842Z 2022-01-10T16:11:59.706928Z 2022-01-10T16:12:03.679714Z 2022-01-10T16:11:59.497692Z 2022-01-10T16:12:05.414445Z 2022-01-10T16:11:58.952463Z 2022-01-10T16:12:00.496984Z 2022-01-10T16:12:02.441278Z 2022-01-10T16:12:01.549653Z 2022-01-10T16:12:01.732907Z 2022-01-10T16:12:03.068472Z 2022-01-10T16:11:59.267583Z 2022-01-10T16:12:00.242134Z 2022-01-10T16:11:59.493180Z 2022-01-10T16:11:50.800708Z 2022-01-10T16:11:52.126356Z 2022-01-10T16:11:55.266073Z 2022-01-10T16:11:56.056433Z 2022-01-10T16:12:02.387235Z 2022-01-10T16:11:56.890489Z 2022-01-10T16:11:58.207719Z 2022-01-10T16:11:52.220041Z 2022-01-10T16:11:57.211694Z 2022-01-10T16:11:49.327545Z 2022-01-10T16:11:55.882049Z 2022-01-10T16:11:47.610603Z 2022-01-10T16:11:48.122050Z 2022-01-10T16:11:43.850415Z 2022-01-10T16:11:53.039176Z 2022-01-10T16:11:46.069441Z 2022-01-10T16:11:25.210554Z 2022-01-10T16:11:45.713763Z 2022-01-10T16:11:42.039420Z 2022-01-10T16:11:52.633772Z 2022-01-10T16:11:22.671474Z 2022-01-10T16:11:46.583868Z 2022-01-10T16:11:51.309411Z 2022-01-10T16:11:24.075772Z 2022-01-10T16:11:22.326737Z 2022-01-10T16:11:24.864138Z 2022-01-10T16:11:23.583427Z 2022-01-10T16:11:21.655837Z 2022-01-10T16:11:22.218659Z 2022-01-10T16:11:24.071320Z 2022-01-10T16:11:22.326737Z 2022-01-10T16:11:20.084802Z 2022-01-10T16:11:20.063197Z 2022-01-10T16:11:24.535816Z 2022-01-10T16:11:21.130272Z 2022-01-10T16:11:22.401887Z 2022-01-10T16:11:18.344469Z 2022-01-10T16:11:16.534764Z 2022-01-10T16:11:21.048979Z 2022-01-10T16:11:17.785418Z 2022-01-10T16:11:19.139860Z 2022-01-10T16:11:17.928738Z 2022-01-10T16:11:13.342587Z 2022-01-10T16:11:18.588764Z 2022-01-10T16:11:15.773429Z 2022-01-10T16:11:16.030785Z 2022-01-10T16:11:12.783729Z 2022-01-10T16:11:14.557334Z 2022-01-10T16:11:13.053428Z 2022-01-10T16:11:12.043321Z 2022-01-10T16:11:16.588430Z 2022-01-10T16:11:11.619738Z 2022-01-10T16:11:11.953039Z 2022-01-10T16:11:16.295811Z 2022-01-10T16:11:12.306451Z 2022-01-10T16:11:12.416711Z 2022-01-10T16:11:09.308981Z 2022-01-10T16:11:04.830574Z 2022-01-10T16:11:10.131458Z 2022-01-10T16:11:05.478713Z 2022-01-10T16:11:08.894436Z 2022-01-10T16:11:06.592709Z 2022-01-10T16:11:11.863489Z 2022-01-10T16:11:03.214715Z 2022-01-10T16:11:03.582458Z 2022-01-10T16:10:46.433139Z 2022-01-10T16:10:43.878427Z 2022-01-10T16:10:44.817718Z 2022-01-10T16:10:46.783347Z 2022-01-10T16:10:45.670868Z 2022-01-10T16:10:43.112400Z 2022-01-10T16:10:41.753548Z 2022-01-10T16:10:34.543820Z 2022-01-10T16:10:38.077861Z", + "failed_job_ids": " []", + "finished": "2022-01-10T15:52:44.876155Z 2022-01-10T15:53:32.703634Z 2022-01-10T15:53:34.503339Z 2022-01-10T15:53:33.521918Z 2022-01-10T15:54:12.255977Z 2022-01-10T15:53:26.250601Z 2022-01-10T15:54:39.275758Z 2022-01-10T15:53:32.190149Z 2022-01-10T15:53:17.703159Z 2022-01-10T15:53:17.996701Z 2022-01-10T15:55:34.801772Z 2022-01-10T15:53:18.419326Z 2022-01-10T15:55:25.682472Z 2022-01-10T15:53:11.934731Z 2022-01-10T15:54:47.643979Z 2022-01-10T15:53:16.064179Z 2022-01-10T15:54:56.380790Z 2022-01-10T15:53:15.298993Z 2022-01-10T15:54:20.954879Z 2022-01-10T15:53:05.361738Z 2022-01-10T15:53:26.142956Z 2022-01-10T15:53:02.676447Z 2022-01-10T15:53:54.374700Z 2022-01-10T15:52:44.448095Z 2022-01-10T15:53:45.101639Z 2022-01-10T15:52:34.792609Z 2022-01-10T15:54:03.834695Z 2022-01-10T15:52:42.767519Z 2022-01-10T15:55:16.671039Z 2022-01-10T15:52:23.800664Z 2022-01-10T15:55:06.552336Z 2022-01-10T15:52:35.010431Z 2022-01-10T15:54:30.265065Z 2022-01-10T15:52:12.719606Z 2022-01-10T15:52:32.731239Z 2022-01-10T15:52:07.344641Z 2022-01-10T15:53:35.622854Z 2022-01-10T15:53:16.985945Z 2022-01-10T15:53:41.558308Z 2022-01-10T15:53:17.509763Z 2022-01-10T15:53:39.685016Z 2022-01-10T15:53:17.863782Z 2022-01-10T15:53:37.847812Z 2022-01-10T15:53:16.080125Z 2022-01-10T15:53:41.647910Z 2022-01-10T15:53:17.316090Z 2022-01-10T15:53:42.874082Z 2022-01-10T15:53:42.288140Z 2022-01-10T15:53:38.047591Z 2022-01-10T15:53:39.571817Z 2022-01-10T15:53:13.194500Z 2022-01-10T15:53:40.831993Z 2022-01-10T15:53:16.968602Z 2022-01-10T15:53:43.124291Z 2022-01-10T15:53:13.652067Z 2022-01-10T15:53:42.746907Z 2022-01-10T15:53:17.245618Z 2022-01-10T15:53:31.367903Z 2022-01-10T15:53:10.008365Z 2022-01-10T15:53:40.981883Z 2022-01-10T15:53:13.952802Z 2022-01-10T15:53:34.819431Z 2022-01-10T15:53:13.652921Z 2022-01-10T15:53:38.043054Z 2022-01-10T15:53:11.981111Z 2022-01-10T15:53:37.571695Z 2022-01-10T15:53:11.692770Z 2022-01-10T15:53:42.632565Z 2022-01-10T15:53:08.715257Z 2022-01-10T15:53:39.577176Z 2022-01-10T15:53:11.569942Z 2022-01-10T15:53:42.812423Z 2022-01-10T15:53:12.762080Z 2022-01-10T15:53:08.533445Z 2022-01-10T15:53:39.541396Z 2022-01-10T15:53:30.763123Z 2022-01-10T15:53:07.576401Z 2022-01-10T15:53:37.545014Z 2022-01-10T15:53:09.342686Z 2022-01-10T15:53:34.207557Z 2022-01-10T15:53:03.134397Z 2022-01-10T15:53:42.547913Z 2022-01-10T15:53:02.458086Z 2022-01-10T15:53:33.554319Z 2022-01-10T15:53:01.842563Z 2022-01-10T15:53:29.068389Z 2022-01-10T15:53:01.970138Z 2022-01-10T15:53:32.527461Z 2022-01-10T15:53:02.714039Z 2022-01-10T15:53:34.501350Z 2022-01-10T15:53:00.746826Z 2022-01-10T15:53:27.689475Z 2022-01-10T15:52:57.692664Z 2022-01-10T15:53:31.354759Z 2022-01-10T15:52:57.018847Z 2022-01-10T15:53:27.279753Z 2022-01-10T15:52:54.058779Z 2022-01-10T15:53:35.321473Z 2022-01-10T15:53:01.807839Z 2022-01-10T15:53:26.509641Z 2022-01-10T15:59:28.035354Z 2022-01-10T15:59:27.108335Z 2022-01-10T15:59:28.526766Z 2022-01-10T15:59:26.886871Z 2022-01-10T15:59:27.482177Z 2022-01-10T15:59:25.456442Z 2022-01-10T15:59:26.937808Z 2022-01-10T15:59:24.738102Z 2022-01-10T15:59:25.630817Z 2022-01-10T15:59:25.131438Z 2022-01-10T15:59:25.447282Z 2022-01-10T15:59:22.323835Z 2022-01-10T15:59:25.289504Z 2022-01-10T15:59:25.661656Z 2022-01-10T15:59:23.625884Z 2022-01-10T15:59:24.095610Z 2022-01-10T15:59:25.702944Z 2022-01-10T15:59:24.470468Z 2022-01-10T15:59:24.932170Z 2022-01-10T15:59:23.235105Z 2022-01-10T15:59:23.885701Z 2022-01-10T15:59:21.650597Z 2022-01-10T15:59:22.849080Z 2022-01-10T15:59:23.187005Z 2022-01-10T15:59:25.788670Z 2022-01-10T15:59:23.847639Z 2022-01-10T15:59:22.030699Z 2022-01-10T15:59:22.172385Z 2022-01-10T15:59:24.491639Z 2022-01-10T15:59:18.829727Z 2022-01-10T15:59:21.852659Z 2022-01-10T15:59:17.313603Z 2022-01-10T15:59:21.837535Z 2022-01-10T15:59:13.782695Z 2022-01-10T15:59:21.866318Z 2022-01-10T15:59:17.311264Z 2022-01-10T15:59:15.271344Z 2022-01-10T15:59:20.101773Z 2022-01-10T15:59:18.781400Z 2022-01-10T15:59:21.696268Z 2022-01-10T15:59:21.257518Z 2022-01-10T15:59:23.012476Z 2022-01-10T15:59:17.604326Z 2022-01-10T15:59:17.760898Z 2022-01-10T15:59:11.249481Z 2022-01-10T15:59:13.051539Z 2022-01-10T15:59:15.882397Z 2022-01-10T15:59:13.037584Z 2022-01-10T15:59:16.167726Z 2022-01-10T15:59:16.718621Z 2022-01-10T15:59:13.692409Z 2022-01-10T15:59:17.234159Z 2022-01-10T15:59:15.385010Z 2022-01-10T15:59:18.792611Z 2022-01-10T15:59:10.412257Z 2022-01-10T15:59:09.574650Z 2022-01-10T15:59:09.435551Z 2022-01-10T15:59:15.929569Z 2022-01-10T15:59:08.551405Z 2022-01-10T15:59:05.517660Z 2022-01-10T15:59:04.036908Z 2022-01-10T15:58:59.387544Z 2022-01-10T15:59:12.858190Z 2022-01-10T15:59:14.626882Z 2022-01-10T15:58:59.622392Z 2022-01-10T15:59:02.534406Z 2022-01-10T15:59:02.429739Z 2022-01-10T15:58:59.501920Z 2022-01-10T15:59:04.234452Z 2022-01-10T15:59:05.139276Z 2022-01-10T15:59:07.877036Z 2022-01-10T15:58:54.073792Z 2022-01-10T15:58:54.156952Z 2022-01-10T15:58:55.612078Z 2022-01-10T15:58:52.096305Z 2022-01-10T15:58:54.767639Z 2022-01-10T15:58:46.688309Z 2022-01-10T15:58:26.089203Z 2022-01-10T15:58:17.588709Z 2022-01-10T15:58:28.414040Z 2022-01-10T15:58:18.224738Z 2022-01-10T15:58:24.284679Z 2022-01-10T15:58:16.168269Z 2022-01-10T15:58:17.443439Z 2022-01-10T15:58:14.871420Z 2022-01-10T15:58:18.685676Z 2022-01-10T15:58:12.341413Z 2022-01-10T15:58:16.501145Z 2022-01-10T15:58:13.351474Z 2022-01-10T15:58:12.924809Z 2022-01-10T15:58:10.463631Z 2022-01-10T15:58:14.580896Z 2022-01-10T15:58:03.859233Z 2022-01-10T15:58:02.929376Z 2022-01-10T15:58:01.998801Z 2022-01-10T15:58:06.349642Z 2022-01-10T15:57:59.973172Z 2022-01-10T15:58:00.207046Z 2022-01-10T15:57:58.246805Z 2022-01-10T15:57:58.103466Z 2022-01-10T16:03:37.536600Z 2022-01-10T16:03:35.407652Z 2022-01-10T16:03:37.108073Z 2022-01-10T16:03:35.509510Z 2022-01-10T16:03:37.741572Z 2022-01-10T16:03:30.798917Z 2022-01-10T16:03:33.879814Z 2022-01-10T16:03:34.842276Z 2022-01-10T16:03:34.968592Z 2022-01-10T16:03:36.321683Z 2022-01-10T16:03:33.880765Z 2022-01-10T16:03:29.596658Z 2022-01-10T16:03:31.395476Z 2022-01-10T16:03:34.206136Z 2022-01-10T16:03:33.990924Z 2022-01-10T16:03:34.945942Z 2022-01-10T16:03:35.411073Z 2022-01-10T16:03:35.133107Z 2022-01-10T16:03:32.461774Z 2022-01-10T16:03:34.220725Z 2022-01-10T16:03:34.262291Z 2022-01-10T16:03:32.300896Z 2022-01-10T16:03:32.332676Z 2022-01-10T16:03:31.181781Z 2022-01-10T16:03:29.998910Z 2022-01-10T16:03:27.906491Z 2022-01-10T16:03:33.241537Z 2022-01-10T16:03:34.452579Z 2022-01-10T16:03:34.867865Z 2022-01-10T16:03:34.978946Z 2022-01-10T16:03:29.314125Z 2022-01-10T16:03:31.446932Z 2022-01-10T16:03:31.314234Z 2022-01-10T16:03:24.008742Z 2022-01-10T16:03:28.540272Z 2022-01-10T16:03:36.211713Z 2022-01-10T16:03:26.282408Z 2022-01-10T16:03:27.728444Z 2022-01-10T16:03:28.483363Z 2022-01-10T16:03:32.172824Z 2022-01-10T16:03:28.892591Z 2022-01-10T16:03:24.611339Z 2022-01-10T16:03:27.083297Z 2022-01-10T16:03:34.612369Z 2022-01-10T16:03:20.960426Z 2022-01-10T16:03:25.112658Z 2022-01-10T16:03:28.806987Z 2022-01-10T16:03:23.060702Z 2022-01-10T16:03:25.584279Z 2022-01-10T16:03:21.261370Z 2022-01-10T16:03:19.613086Z 2022-01-10T16:03:22.938705Z 2022-01-10T16:03:21.394333Z 2022-01-10T16:03:22.489243Z 2022-01-10T16:03:27.977488Z 2022-01-10T16:03:27.350624Z 2022-01-10T16:03:23.544030Z 2022-01-10T16:03:25.715923Z 2022-01-10T16:03:22.704632Z 2022-01-10T16:03:15.955445Z 2022-01-10T16:03:21.918901Z 2022-01-10T16:03:11.294791Z 2022-01-10T16:03:15.639434Z 2022-01-10T16:03:17.355681Z 2022-01-10T16:03:12.550392Z 2022-01-10T16:03:12.568680Z 2022-01-10T16:03:13.030452Z 2022-01-10T16:03:15.516664Z 2022-01-10T16:03:13.593555Z 2022-01-10T16:03:14.261104Z 2022-01-10T16:03:11.528245Z 2022-01-10T16:03:04.441437Z 2022-01-10T16:03:05.949656Z 2022-01-10T16:02:49.979048Z 2022-01-10T16:03:07.934848Z 2022-01-10T16:02:52.158687Z 2022-01-10T16:03:02.142141Z 2022-01-10T16:03:08.759703Z 2022-01-10T16:02:59.222387Z 2022-01-10T16:02:29.489892Z 2022-01-10T16:02:22.021309Z 2022-01-10T16:02:46.767691Z 2022-01-10T16:02:25.636597Z 2022-01-10T16:02:22.470056Z 2022-01-10T16:02:27.899481Z 2022-01-10T16:02:22.205110Z 2022-01-10T16:02:25.982649Z 2022-01-10T16:02:21.039063Z 2022-01-10T16:02:22.173457Z 2022-01-10T16:02:22.732331Z 2022-01-10T16:02:21.058662Z 2022-01-10T16:02:20.121206Z 2022-01-10T16:02:20.033479Z 2022-01-10T16:02:21.092490Z 2022-01-10T16:02:12.864690Z 2022-01-10T16:02:14.687418Z 2022-01-10T16:02:11.010742Z 2022-01-10T16:02:10.147484Z 2022-01-10T16:02:07.773872Z 2022-01-10T16:02:06.508534Z 2022-01-10T16:07:49.567627Z 2022-01-10T16:07:52.209247Z 2022-01-10T16:07:49.357495Z 2022-01-10T16:07:51.914255Z 2022-01-10T16:07:48.468158Z 2022-01-10T16:07:53.020238Z 2022-01-10T16:07:52.299940Z 2022-01-10T16:07:49.902454Z 2022-01-10T16:07:48.366376Z 2022-01-10T16:07:51.168980Z 2022-01-10T16:07:47.525822Z 2022-01-10T16:07:52.208562Z 2022-01-10T16:07:47.562956Z 2022-01-10T16:07:52.563306Z 2022-01-10T16:07:47.438947Z 2022-01-10T16:07:52.345842Z 2022-01-10T16:07:47.733066Z 2022-01-10T16:07:48.841370Z 2022-01-10T16:07:48.896988Z 2022-01-10T16:07:53.312635Z 2022-01-10T16:07:43.729683Z 2022-01-10T16:07:51.930683Z 2022-01-10T16:07:48.215764Z 2022-01-10T16:07:48.190450Z 2022-01-10T16:07:46.972834Z 2022-01-10T16:07:46.957055Z 2022-01-10T16:07:44.271976Z 2022-01-10T16:07:47.144737Z 2022-01-10T16:07:45.070818Z 2022-01-10T16:07:43.080153Z 2022-01-10T16:07:45.208806Z 2022-01-10T16:07:50.801733Z 2022-01-10T16:07:44.052101Z 2022-01-10T16:07:44.845382Z 2022-01-10T16:07:47.464488Z 2022-01-10T16:07:47.641587Z 2022-01-10T16:07:42.892799Z 2022-01-10T16:07:40.572189Z 2022-01-10T16:07:45.583827Z 2022-01-10T16:07:50.008678Z 2022-01-10T16:07:38.108469Z 2022-01-10T16:07:48.229414Z 2022-01-10T16:07:36.181892Z 2022-01-10T16:07:46.078583Z 2022-01-10T16:07:43.961048Z 2022-01-10T16:07:42.044360Z 2022-01-10T16:07:44.506767Z 2022-01-10T16:07:36.305640Z 2022-01-10T16:07:35.944595Z 2022-01-10T16:07:49.347782Z 2022-01-10T16:07:41.194600Z 2022-01-10T16:07:46.683907Z 2022-01-10T16:07:35.841605Z 2022-01-10T16:07:38.482029Z 2022-01-10T16:07:33.748627Z 2022-01-10T16:07:36.040159Z 2022-01-10T16:07:38.818842Z 2022-01-10T16:07:38.416434Z 2022-01-10T16:07:32.917451Z 2022-01-10T16:07:44.442404Z 2022-01-10T16:07:35.852603Z 2022-01-10T16:07:39.029663Z 2022-01-10T16:07:30.473642Z 2022-01-10T16:07:36.624623Z 2022-01-10T16:07:29.259665Z 2022-01-10T16:07:36.197196Z 2022-01-10T16:07:22.634196Z 2022-01-10T16:07:34.932450Z 2022-01-10T16:07:29.283552Z 2022-01-10T16:07:26.048515Z 2022-01-10T16:07:19.436825Z 2022-01-10T16:07:35.112330Z 2022-01-10T16:07:12.346420Z 2022-01-10T16:06:58.825025Z 2022-01-10T16:07:04.463362Z 2022-01-10T16:06:52.004200Z 2022-01-10T16:06:56.980384Z 2022-01-10T16:07:00.991706Z 2022-01-10T16:07:09.696112Z 2022-01-10T16:06:50.670646Z 2022-01-10T16:07:02.724903Z 2022-01-10T16:06:47.639443Z 2022-01-10T16:06:52.543347Z 2022-01-10T16:06:58.965071Z 2022-01-10T16:06:51.027435Z 2022-01-10T16:06:44.586341Z 2022-01-10T16:06:44.449491Z 2022-01-10T16:06:45.772191Z 2022-01-10T16:06:39.711147Z 2022-01-10T16:06:42.959050Z 2022-01-10T16:06:40.292829Z 2022-01-10T16:06:37.350727Z 2022-01-10T16:06:43.282578Z 2022-01-10T16:06:37.106642Z 2022-01-10T16:06:37.911890Z 2022-01-10T16:06:27.916157Z 2022-01-10T16:06:25.638553Z 2022-01-10T16:06:27.349576Z 2022-01-10T16:06:22.954440Z 2022-01-10T16:06:25.043546Z 2022-01-10T16:12:10.119115Z 2022-01-10T16:12:06.092452Z 2022-01-10T16:12:09.881763Z 2022-01-10T16:12:06.326819Z 2022-01-10T16:12:08.877448Z 2022-01-10T16:12:01.283892Z 2022-01-10T16:12:08.476401Z 2022-01-10T16:12:05.508520Z 2022-01-10T16:12:07.087283Z 2022-01-10T16:12:04.397729Z 2022-01-10T16:12:08.563780Z 2022-01-10T16:12:05.152590Z 2022-01-10T16:12:08.092499Z 2022-01-10T16:12:04.067186Z 2022-01-10T16:12:08.287077Z 2022-01-10T16:12:04.139667Z 2022-01-10T16:12:08.814732Z 2022-01-10T16:12:03.417774Z 2022-01-10T16:12:04.189699Z 2022-01-10T16:12:06.962936Z 2022-01-10T16:12:05.322172Z 2022-01-10T16:12:05.398862Z 2022-01-10T16:12:06.667484Z 2022-01-10T16:12:03.962153Z 2022-01-10T16:12:05.397690Z 2022-01-10T16:12:04.311030Z 2022-01-10T16:11:55.280440Z 2022-01-10T16:11:57.000669Z 2022-01-10T16:12:00.854559Z 2022-01-10T16:11:59.970332Z 2022-01-10T16:12:06.777840Z 2022-01-10T16:12:01.957708Z 2022-01-10T16:12:02.231839Z 2022-01-10T16:11:56.902560Z 2022-01-10T16:12:01.408585Z 2022-01-10T16:11:53.789164Z 2022-01-10T16:12:00.297518Z 2022-01-10T16:11:52.617307Z 2022-01-10T16:11:52.931132Z 2022-01-10T16:11:48.685235Z 2022-01-10T16:11:58.438074Z 2022-01-10T16:11:50.605699Z 2022-01-10T16:11:29.610001Z 2022-01-10T16:11:50.585784Z 2022-01-10T16:11:47.353249Z 2022-01-10T16:11:57.887391Z 2022-01-10T16:11:27.377029Z 2022-01-10T16:11:51.547468Z 2022-01-10T16:11:56.026602Z 2022-01-10T16:11:28.640705Z 2022-01-10T16:11:27.148625Z 2022-01-10T16:11:29.147369Z 2022-01-10T16:11:27.899476Z 2022-01-10T16:11:26.110299Z 2022-01-10T16:11:27.082062Z 2022-01-10T16:11:28.790302Z 2022-01-10T16:11:26.691093Z 2022-01-10T16:11:24.188580Z 2022-01-10T16:11:24.729044Z 2022-01-10T16:11:29.073981Z 2022-01-10T16:11:25.922183Z 2022-01-10T16:11:26.707306Z 2022-01-10T16:11:23.857746Z 2022-01-10T16:11:21.834255Z 2022-01-10T16:11:25.946300Z 2022-01-10T16:11:21.415662Z 2022-01-10T16:11:23.656804Z 2022-01-10T16:11:22.398776Z 2022-01-10T16:11:18.083127Z 2022-01-10T16:11:22.993996Z 2022-01-10T16:11:20.581289Z 2022-01-10T16:11:22.136626Z 2022-01-10T16:11:17.127595Z 2022-01-10T16:11:19.871633Z 2022-01-10T16:11:17.781814Z 2022-01-10T16:11:16.544846Z 2022-01-10T16:11:20.426648Z 2022-01-10T16:11:17.165697Z 2022-01-10T16:11:17.419802Z 2022-01-10T16:11:21.365597Z 2022-01-10T16:11:17.138777Z 2022-01-10T16:11:17.871198Z 2022-01-10T16:11:14.370694Z 2022-01-10T16:11:10.105614Z 2022-01-10T16:11:15.733339Z 2022-01-10T16:11:11.161672Z 2022-01-10T16:11:13.188269Z 2022-01-10T16:11:10.875592Z 2022-01-10T16:11:17.318255Z 2022-01-10T16:11:08.051622Z 2022-01-10T16:11:08.517177Z 2022-01-10T16:10:51.012759Z 2022-01-10T16:10:47.914245Z 2022-01-10T16:10:49.352050Z 2022-01-10T16:10:50.718571Z 2022-01-10T16:10:50.164075Z 2022-01-10T16:10:46.871363Z 2022-01-10T16:10:46.044396Z 2022-01-10T16:10:38.757763Z 2022-01-10T16:10:39.827603Z", + "finished_diff": 114.16660320000001, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 210.223588, + "mean": 50.96211663399997, + "min": 11.114076 + }, + "jobs_events_duration": { + "max": 6.945152, + "mean": 2.847008128, + "min": 0.144241 + }, + "jobs_events_lag": { + "max": -1.693643, + "mean": -4.844460086000002, + "min": -7.74571 + }, + "jobs_waiting": { + "max": 56.155783, + "mean": 23.291422070000014, + "min": 0.616494 + }, + "modified": "2022-01-10T15:52:07.558956Z 2022-01-10T15:52:07.509978Z 2022-01-10T15:52:01.260259Z 2022-01-10T15:52:01.152803Z 2022-01-10T15:52:01.092713Z 2022-01-10T15:52:00.984876Z 2022-01-10T15:52:00.808553Z 2022-01-10T15:52:00.596821Z 2022-01-10T15:52:00.403519Z 2022-01-10T15:52:00.261107Z 2022-01-10T15:52:00.133435Z 2022-01-10T15:52:00.034789Z 2022-01-10T15:51:59.906366Z 2022-01-10T15:51:59.752036Z 2022-01-10T15:51:59.611424Z 2022-01-10T15:51:59.522299Z 2022-01-10T15:51:59.385227Z 2022-01-10T15:51:59.319651Z 2022-01-10T15:51:59.268797Z 2022-01-10T15:51:59.134026Z 2022-01-10T15:51:59.014856Z 2022-01-10T15:51:58.841389Z 2022-01-10T15:51:56.122726Z 2022-01-10T15:51:56.040438Z 2022-01-10T15:51:55.962065Z 2022-01-10T15:51:55.854039Z 2022-01-10T15:51:55.788090Z 2022-01-10T15:51:55.719114Z 2022-01-10T15:51:55.637000Z 2022-01-10T15:51:55.569625Z 2022-01-10T15:51:55.515574Z 2022-01-10T15:51:55.428845Z 2022-01-10T15:51:54.297072Z 2022-01-10T15:51:54.231673Z 2022-01-10T15:51:54.144679Z 2022-01-10T15:51:53.512712Z 2022-01-10T15:52:27.851500Z 2022-01-10T15:52:27.735053Z 2022-01-10T15:52:27.628697Z 2022-01-10T15:52:27.489609Z 2022-01-10T15:52:27.378980Z 2022-01-10T15:52:27.270430Z 2022-01-10T15:52:27.165309Z 2022-01-10T15:52:27.044294Z 2022-01-10T15:52:26.912333Z 2022-01-10T15:52:26.825341Z 2022-01-10T15:52:26.735337Z 2022-01-10T15:52:26.652519Z 2022-01-10T15:52:26.550850Z 2022-01-10T15:52:09.474382Z 2022-01-10T15:52:09.435355Z 2022-01-10T15:52:09.395032Z 2022-01-10T15:52:09.359582Z 2022-01-10T15:52:09.325147Z 2022-01-10T15:52:09.291324Z 2022-01-10T15:52:09.256874Z 2022-01-10T15:52:09.223934Z 2022-01-10T15:52:09.175620Z 2022-01-10T15:52:09.076067Z 2022-01-10T15:52:09.009929Z 2022-01-10T15:52:08.944625Z 2022-01-10T15:52:08.899080Z 2022-01-10T15:52:08.858114Z 2022-01-10T15:52:08.824538Z 2022-01-10T15:52:08.789780Z 2022-01-10T15:52:08.757740Z 2022-01-10T15:52:08.724546Z 2022-01-10T15:52:08.691907Z 2022-01-10T15:52:08.656607Z 2022-01-10T15:52:08.621085Z 2022-01-10T15:52:08.580721Z 2022-01-10T15:52:08.547164Z 2022-01-10T15:52:08.514591Z 2022-01-10T15:52:08.449316Z 2022-01-10T15:52:08.482653Z 2022-01-10T15:52:08.415971Z 2022-01-10T15:52:08.382425Z 2022-01-10T15:52:08.349718Z 2022-01-10T15:52:08.317127Z 2022-01-10T15:52:08.284463Z 2022-01-10T15:52:08.251831Z 2022-01-10T15:52:08.217953Z 2022-01-10T15:52:08.182764Z 2022-01-10T15:52:08.148283Z 2022-01-10T15:52:08.112614Z 2022-01-10T15:52:08.079354Z 2022-01-10T15:52:08.046618Z 2022-01-10T15:52:08.013946Z 2022-01-10T15:52:07.980252Z 2022-01-10T15:52:07.946786Z 2022-01-10T15:52:07.911964Z 2022-01-10T15:52:07.880112Z 2022-01-10T15:52:07.846129Z 2022-01-10T15:52:07.810964Z 2022-01-10T15:52:07.776985Z 2022-01-10T15:52:07.744370Z 2022-01-10T15:52:07.711550Z 2022-01-10T15:52:07.678147Z 2022-01-10T15:52:07.638609Z 2022-01-10T15:52:07.595087Z 2022-01-10T15:58:48.740762Z 2022-01-10T15:58:48.654541Z 2022-01-10T15:58:48.569436Z 2022-01-10T15:58:48.489258Z 2022-01-10T15:58:48.409851Z 2022-01-10T15:58:08.209137Z 2022-01-10T15:58:08.166370Z 2022-01-10T15:58:08.115502Z 2022-01-10T15:58:08.064534Z 2022-01-10T15:58:07.998573Z 2022-01-10T15:58:07.941643Z 2022-01-10T15:58:07.899925Z 2022-01-10T15:58:07.852813Z 2022-01-10T15:58:07.735453Z 2022-01-10T15:58:07.645374Z 2022-01-10T15:58:07.546652Z 2022-01-10T15:58:07.373559Z 2022-01-10T15:58:07.267870Z 2022-01-10T15:58:07.195680Z 2022-01-10T15:58:07.162593Z 2022-01-10T15:58:07.128272Z 2022-01-10T15:58:07.095348Z 2022-01-10T15:58:06.976614Z 2022-01-10T15:58:06.939372Z 2022-01-10T15:58:06.902784Z 2022-01-10T15:58:06.866399Z 2022-01-10T15:58:06.822387Z 2022-01-10T15:58:06.788522Z 2022-01-10T15:58:06.750892Z 2022-01-10T15:58:06.717270Z 2022-01-10T15:58:06.683165Z 2022-01-10T15:58:06.649724Z 2022-01-10T15:58:06.608229Z 2022-01-10T15:58:06.574586Z 2022-01-10T15:58:06.520468Z 2022-01-10T15:58:06.477274Z 2022-01-10T15:58:06.431457Z 2022-01-10T15:58:06.387657Z 2022-01-10T15:58:06.329112Z 2022-01-10T15:58:06.274568Z 2022-01-10T15:58:06.210705Z 2022-01-10T15:58:06.137092Z 2022-01-10T15:58:06.047433Z 2022-01-10T15:58:05.960545Z 2022-01-10T15:58:05.844624Z 2022-01-10T15:58:05.733567Z 2022-01-10T15:58:05.627479Z 2022-01-10T15:58:05.502706Z 2022-01-10T15:58:05.381520Z 2022-01-10T15:58:05.233922Z 2022-01-10T15:58:05.180766Z 2022-01-10T15:58:05.132709Z 2022-01-10T15:58:05.087551Z 2022-01-10T15:58:05.053077Z 2022-01-10T15:58:05.003812Z 2022-01-10T15:58:04.935809Z 2022-01-10T15:58:04.890995Z 2022-01-10T15:58:04.857811Z 2022-01-10T15:58:04.807478Z 2022-01-10T15:58:04.737829Z 2022-01-10T15:58:04.671391Z 2022-01-10T15:58:04.629793Z 2022-01-10T15:58:04.591834Z 2022-01-10T15:58:04.524606Z 2022-01-10T15:58:04.477162Z 2022-01-10T15:58:04.405418Z 2022-01-10T15:58:04.275632Z 2022-01-10T15:58:04.126869Z 2022-01-10T15:58:03.982397Z 2022-01-10T15:58:03.941622Z 2022-01-10T15:58:03.907886Z 2022-01-10T15:58:03.869528Z 2022-01-10T15:58:03.810416Z 2022-01-10T15:58:03.711501Z 2022-01-10T15:58:03.629369Z 2022-01-10T15:58:03.563545Z 2022-01-10T15:58:03.491397Z 2022-01-10T15:57:50.712919Z 2022-01-10T15:57:50.349260Z 2022-01-10T15:57:50.125390Z 2022-01-10T15:57:49.893561Z 2022-01-10T15:57:49.744539Z 2022-01-10T15:57:49.612663Z 2022-01-10T15:57:49.499241Z 2022-01-10T15:57:49.364031Z 2022-01-10T15:57:49.179495Z 2022-01-10T15:57:48.992998Z 2022-01-10T15:57:48.770936Z 2022-01-10T15:57:48.496985Z 2022-01-10T15:57:48.279115Z 2022-01-10T15:57:48.141413Z 2022-01-10T15:57:47.911598Z 2022-01-10T15:57:45.328992Z 2022-01-10T15:57:45.275632Z 2022-01-10T15:57:45.156751Z 2022-01-10T15:57:45.091046Z 2022-01-10T15:57:44.224785Z 2022-01-10T15:57:44.182145Z 2022-01-10T15:57:43.799124Z 2022-01-10T15:57:43.752929Z 2022-01-10T16:02:48.439184Z 2022-01-10T16:02:48.342789Z 2022-01-10T16:02:48.234758Z 2022-01-10T16:02:14.237033Z 2022-01-10T16:02:48.111782Z 2022-01-10T16:02:14.165619Z 2022-01-10T16:02:14.106362Z 2022-01-10T16:02:14.061889Z 2022-01-10T16:02:14.026714Z 2022-01-10T16:02:13.964612Z 2022-01-10T16:02:13.920464Z 2022-01-10T16:02:13.882079Z 2022-01-10T16:02:13.842499Z 2022-01-10T16:02:13.807424Z 2022-01-10T16:02:13.774327Z 2022-01-10T16:02:13.735185Z 2022-01-10T16:02:13.701958Z 2022-01-10T16:02:13.664253Z 2022-01-10T16:02:13.629125Z 2022-01-10T16:02:13.590413Z 2022-01-10T16:02:13.540405Z 2022-01-10T16:02:13.447733Z 2022-01-10T16:02:13.345649Z 2022-01-10T16:02:13.284340Z 2022-01-10T16:02:13.220620Z 2022-01-10T16:02:13.143206Z 2022-01-10T16:02:13.027489Z 2022-01-10T16:02:12.910388Z 2022-01-10T16:02:12.827819Z 2022-01-10T16:02:12.794135Z 2022-01-10T16:02:12.759841Z 2022-01-10T16:02:12.719944Z 2022-01-10T16:02:12.681746Z 2022-01-10T16:02:12.645834Z 2022-01-10T16:02:12.611773Z 2022-01-10T16:02:12.573591Z 2022-01-10T16:02:12.535704Z 2022-01-10T16:02:12.500178Z 2022-01-10T16:02:12.465009Z 2022-01-10T16:02:12.420521Z 2022-01-10T16:02:12.383352Z 2022-01-10T16:02:12.342843Z 2022-01-10T16:02:12.309325Z 2022-01-10T16:02:12.275882Z 2022-01-10T16:02:12.243440Z 2022-01-10T16:02:12.210901Z 2022-01-10T16:02:12.174987Z 2022-01-10T16:02:12.132552Z 2022-01-10T16:02:12.098569Z 2022-01-10T16:02:12.063933Z 2022-01-10T16:02:12.025129Z 2022-01-10T16:02:11.990307Z 2022-01-10T16:02:11.926320Z 2022-01-10T16:02:11.862356Z 2022-01-10T16:02:11.765603Z 2022-01-10T16:02:11.654338Z 2022-01-10T16:02:11.543435Z 2022-01-10T16:02:11.470484Z 2022-01-10T16:02:11.429973Z 2022-01-10T16:02:11.299413Z 2022-01-10T16:02:11.192513Z 2022-01-10T16:02:11.079426Z 2022-01-10T16:02:10.971851Z 2022-01-10T16:02:10.876504Z 2022-01-10T16:02:10.775310Z 2022-01-10T16:02:10.694365Z 2022-01-10T16:02:10.610318Z 2022-01-10T16:02:10.529813Z 2022-01-10T16:02:10.485312Z 2022-01-10T16:02:10.444305Z 2022-01-10T16:02:10.399303Z 2022-01-10T16:02:10.353134Z 2022-01-10T16:02:10.308843Z 2022-01-10T16:02:10.263174Z 2022-01-10T16:02:10.220592Z 2022-01-10T16:02:10.178163Z 2022-01-10T16:02:10.104461Z 2022-01-10T16:02:09.978598Z 2022-01-10T16:02:09.866080Z 2022-01-10T16:01:59.130216Z 2022-01-10T16:01:58.926061Z 2022-01-10T16:02:09.706792Z 2022-01-10T16:01:58.794940Z 2022-01-10T16:01:58.589045Z 2022-01-10T16:01:58.327425Z 2022-01-10T16:01:58.107107Z 2022-01-10T16:01:58.024083Z 2022-01-10T16:01:57.938699Z 2022-01-10T16:01:57.843861Z 2022-01-10T16:01:57.655775Z 2022-01-10T16:01:57.481497Z 2022-01-10T16:01:56.981474Z 2022-01-10T16:01:56.432305Z 2022-01-10T16:01:56.084323Z 2022-01-10T16:01:54.312926Z 2022-01-10T16:01:54.249349Z 2022-01-10T16:01:54.199859Z 2022-01-10T16:01:53.576962Z 2022-01-10T16:01:53.001875Z 2022-01-10T16:01:52.966233Z 2022-01-10T16:06:31.540349Z 2022-01-10T16:06:31.452956Z 2022-01-10T16:06:31.337473Z 2022-01-10T16:06:31.237329Z 2022-01-10T16:06:31.145445Z 2022-01-10T16:06:31.061256Z 2022-01-10T16:06:30.882867Z 2022-01-10T16:06:30.969440Z 2022-01-10T16:06:30.822529Z 2022-01-10T16:06:30.763433Z 2022-01-10T16:06:30.704259Z 2022-01-10T16:06:30.654369Z 2022-01-10T16:06:30.602357Z 2022-01-10T16:06:30.542274Z 2022-01-10T16:06:30.488307Z 2022-01-10T16:06:30.432284Z 2022-01-10T16:06:30.377249Z 2022-01-10T16:06:30.322330Z 2022-01-10T16:06:30.265615Z 2022-01-10T16:06:30.212310Z 2022-01-10T16:06:30.167015Z 2022-01-10T16:06:30.116034Z 2022-01-10T16:06:30.067275Z 2022-01-10T16:06:30.021271Z 2022-01-10T16:06:29.970384Z 2022-01-10T16:06:29.918426Z 2022-01-10T16:06:29.859378Z 2022-01-10T16:06:29.805297Z 2022-01-10T16:06:29.746415Z 2022-01-10T16:06:29.689396Z 2022-01-10T16:06:29.634338Z 2022-01-10T16:06:29.588839Z 2022-01-10T16:06:29.536510Z 2022-01-10T16:06:29.478597Z 2022-01-10T16:06:29.419375Z 2022-01-10T16:06:29.368389Z 2022-01-10T16:06:29.315701Z 2022-01-10T16:06:29.265364Z 2022-01-10T16:06:29.217118Z 2022-01-10T16:06:29.157349Z 2022-01-10T16:06:29.109698Z 2022-01-10T16:06:29.035426Z 2022-01-10T16:06:28.965276Z 2022-01-10T16:06:28.923350Z 2022-01-10T16:06:28.852580Z 2022-01-10T16:06:28.781291Z 2022-01-10T16:06:28.738316Z 2022-01-10T16:06:28.697385Z 2022-01-10T16:06:28.650289Z 2022-01-10T16:06:28.602150Z 2022-01-10T16:06:28.552393Z 2022-01-10T16:06:28.507271Z 2022-01-10T16:06:28.443386Z 2022-01-10T16:06:28.353306Z 2022-01-10T16:06:28.266542Z 2022-01-10T16:06:28.159353Z 2022-01-10T16:06:28.038172Z 2022-01-10T16:06:27.936406Z 2022-01-10T16:06:27.823639Z 2022-01-10T16:06:27.708570Z 2022-01-10T16:06:27.612445Z 2022-01-10T16:06:27.518477Z 2022-01-10T16:06:27.424392Z 2022-01-10T16:06:27.337374Z 2022-01-10T16:06:27.248734Z 2022-01-10T16:06:27.118488Z 2022-01-10T16:06:26.998482Z 2022-01-10T16:06:26.884370Z 2022-01-10T16:06:26.737481Z 2022-01-10T16:06:26.616532Z 2022-01-10T16:06:26.519487Z 2022-01-10T16:06:26.407783Z 2022-01-10T16:06:17.305061Z 2022-01-10T16:06:17.184076Z 2022-01-10T16:06:17.078106Z 2022-01-10T16:06:16.950204Z 2022-01-10T16:06:16.827045Z 2022-01-10T16:06:16.770859Z 2022-01-10T16:06:16.684175Z 2022-01-10T16:06:16.601420Z 2022-01-10T16:06:16.420552Z 2022-01-10T16:06:16.213764Z 2022-01-10T16:06:15.925772Z 2022-01-10T16:06:15.623608Z 2022-01-10T16:06:15.325721Z 2022-01-10T16:06:15.058958Z 2022-01-10T16:06:14.675697Z 2022-01-10T16:06:14.305820Z 2022-01-10T16:06:14.181050Z 2022-01-10T16:06:14.013954Z 2022-01-10T16:06:13.578928Z 2022-01-10T16:06:13.107471Z 2022-01-10T16:06:12.887765Z 2022-01-10T16:06:12.763698Z 2022-01-10T16:06:12.663981Z 2022-01-10T16:06:08.677056Z 2022-01-10T16:06:08.634760Z 2022-01-10T16:06:08.597336Z 2022-01-10T16:06:08.554973Z 2022-01-10T16:06:08.519173Z 2022-01-10T16:11:15.542595Z 2022-01-10T16:11:15.451799Z 2022-01-10T16:11:15.337939Z 2022-01-10T16:11:15.214744Z 2022-01-10T16:11:15.103759Z 2022-01-10T16:11:14.984803Z 2022-01-10T16:11:14.876603Z 2022-01-10T16:11:14.762686Z 2022-01-10T16:11:14.657835Z 2022-01-10T16:11:14.540296Z 2022-01-10T16:11:14.474727Z 2022-01-10T16:11:14.386843Z 2022-01-10T16:11:14.300551Z 2022-01-10T16:11:14.232932Z 2022-01-10T16:11:14.166678Z 2022-01-10T16:11:14.087956Z 2022-01-10T16:11:14.008614Z 2022-01-10T16:11:13.945608Z 2022-01-10T16:11:13.883719Z 2022-01-10T16:11:13.769197Z 2022-01-10T16:11:13.645846Z 2022-01-10T16:11:13.533619Z 2022-01-10T16:11:13.434866Z 2022-01-10T16:11:13.309717Z 2022-01-10T16:11:13.172253Z 2022-01-10T16:11:12.944983Z 2022-01-10T16:11:12.734006Z 2022-01-10T16:11:12.677773Z 2022-01-10T16:11:12.633016Z 2022-01-10T16:11:12.558940Z 2022-01-10T16:11:12.454856Z 2022-01-10T16:11:12.350128Z 2022-01-10T16:11:12.281636Z 2022-01-10T16:11:12.209901Z 2022-01-10T16:11:12.113006Z 2022-01-10T16:11:11.952738Z 2022-01-10T16:11:11.800712Z 2022-01-10T16:11:11.616842Z 2022-01-10T16:11:11.413357Z 2022-01-10T16:11:11.246081Z 2022-01-10T16:11:11.088081Z 2022-01-10T16:11:10.963896Z 2022-01-10T16:10:40.384061Z 2022-01-10T16:11:10.866894Z 2022-01-10T16:11:10.706809Z 2022-01-10T16:11:10.520772Z 2022-01-10T16:10:40.348907Z 2022-01-10T16:11:10.440689Z 2022-01-10T16:11:10.346988Z 2022-01-10T16:10:40.315363Z 2022-01-10T16:10:40.283938Z 2022-01-10T16:10:40.241858Z 2022-01-10T16:10:40.208483Z 2022-01-10T16:10:40.173079Z 2022-01-10T16:10:40.136050Z 2022-01-10T16:10:40.102355Z 2022-01-10T16:10:40.066806Z 2022-01-10T16:10:40.033036Z 2022-01-10T16:10:39.995271Z 2022-01-10T16:10:39.960262Z 2022-01-10T16:10:39.924448Z 2022-01-10T16:10:39.891427Z 2022-01-10T16:10:39.858860Z 2022-01-10T16:10:39.825871Z 2022-01-10T16:10:39.792315Z 2022-01-10T16:10:39.759676Z 2022-01-10T16:10:39.705621Z 2022-01-10T16:10:39.666041Z 2022-01-10T16:10:39.627995Z 2022-01-10T16:10:39.592471Z 2022-01-10T16:10:39.551204Z 2022-01-10T16:10:39.504680Z 2022-01-10T16:10:39.381713Z 2022-01-10T16:10:39.298689Z 2022-01-10T16:10:39.162818Z 2022-01-10T16:10:39.045380Z 2022-01-10T16:10:38.987626Z 2022-01-10T16:10:38.933651Z 2022-01-10T16:10:38.872166Z 2022-01-10T16:10:38.809550Z 2022-01-10T16:10:38.747401Z 2022-01-10T16:10:38.674957Z 2022-01-10T16:10:38.521626Z 2022-01-10T16:10:38.391781Z 2022-01-10T16:10:38.217174Z 2022-01-10T16:10:38.083340Z 2022-01-10T16:10:38.014946Z 2022-01-10T16:10:37.928019Z 2022-01-10T16:10:37.755872Z 2022-01-10T16:10:37.486198Z 2022-01-10T16:10:37.286596Z 2022-01-10T16:10:29.606628Z 2022-01-10T16:10:29.493151Z 2022-01-10T16:10:29.365539Z 2022-01-10T16:10:29.248712Z 2022-01-10T16:10:29.160413Z 2022-01-10T16:10:29.045407Z 2022-01-10T16:10:28.961842Z 2022-01-10T16:10:27.381072Z 2022-01-10T16:10:27.346276Z", + "modified_diff": 45.2034134, + "started": "2022-01-10T15:52:09.795559Z 2022-01-10T15:52:10.975555Z 2022-01-10T15:52:06.623114Z 2022-01-10T15:52:07.842973Z 2022-01-10T15:52:06.276823Z 2022-01-10T15:52:06.773710Z 2022-01-10T15:52:05.788029Z 2022-01-10T15:52:07.732478Z 2022-01-10T15:52:05.469751Z 2022-01-10T15:52:06.542314Z 2022-01-10T15:52:04.578184Z 2022-01-10T15:52:04.553276Z 2022-01-10T15:52:04.206969Z 2022-01-10T15:52:04.443157Z 2022-01-10T15:52:03.907394Z 2022-01-10T15:52:03.644680Z 2022-01-10T15:52:03.030256Z 2022-01-10T15:52:02.893517Z 2022-01-10T15:52:02.436450Z 2022-01-10T15:52:02.759867Z 2022-01-10T15:52:02.019379Z 2022-01-10T15:52:02.542532Z 2022-01-10T15:51:58.033670Z 2022-01-10T15:51:57.957296Z 2022-01-10T15:51:57.541369Z 2022-01-10T15:51:57.428642Z 2022-01-10T15:51:57.273959Z 2022-01-10T15:51:57.325493Z 2022-01-10T15:51:56.863287Z 2022-01-10T15:51:56.658683Z 2022-01-10T15:51:56.650346Z 2022-01-10T15:51:56.522568Z 2022-01-10T15:51:55.136352Z 2022-01-10T15:51:55.005264Z 2022-01-10T15:51:54.564291Z 2022-01-10T15:51:53.722921Z 2022-01-10T15:52:31.075731Z 2022-01-10T15:52:30.372195Z 2022-01-10T15:52:30.717560Z 2022-01-10T15:52:29.987310Z 2022-01-10T15:52:29.760592Z 2022-01-10T15:52:29.682075Z 2022-01-10T15:52:29.915179Z 2022-01-10T15:52:29.297723Z 2022-01-10T15:52:29.612539Z 2022-01-10T15:52:28.836392Z 2022-01-10T15:52:29.380860Z 2022-01-10T15:52:28.499768Z 2022-01-10T15:52:28.380119Z 2022-01-10T15:52:23.957328Z 2022-01-10T15:52:23.648076Z 2022-01-10T15:52:23.801680Z 2022-01-10T15:52:23.005015Z 2022-01-10T15:52:23.247627Z 2022-01-10T15:52:22.313155Z 2022-01-10T15:52:22.267698Z 2022-01-10T15:52:21.741470Z 2022-01-10T15:52:22.221585Z 2022-01-10T15:52:21.039280Z 2022-01-10T15:52:21.965551Z 2022-01-10T15:52:20.414147Z 2022-01-10T15:52:20.120380Z 2022-01-10T15:52:19.351923Z 2022-01-10T15:52:20.205931Z 2022-01-10T15:52:19.123515Z 2022-01-10T15:52:20.387009Z 2022-01-10T15:52:18.203549Z 2022-01-10T15:52:18.929752Z 2022-01-10T15:52:17.567358Z 2022-01-10T15:52:18.449839Z 2022-01-10T15:52:16.767634Z 2022-01-10T15:52:17.565821Z 2022-01-10T15:52:16.238095Z 2022-01-10T15:52:15.755284Z 2022-01-10T15:52:16.609577Z 2022-01-10T15:52:16.400613Z 2022-01-10T15:52:15.348181Z 2022-01-10T15:52:15.430863Z 2022-01-10T15:52:14.510646Z 2022-01-10T15:52:14.232533Z 2022-01-10T15:52:14.076923Z 2022-01-10T15:52:13.715366Z 2022-01-10T15:52:12.837813Z 2022-01-10T15:52:12.519185Z 2022-01-10T15:52:12.239939Z 2022-01-10T15:52:12.433779Z 2022-01-10T15:52:11.988051Z 2022-01-10T15:52:12.253834Z 2022-01-10T15:52:11.797689Z 2022-01-10T15:52:11.684392Z 2022-01-10T15:52:11.245146Z 2022-01-10T15:52:11.505495Z 2022-01-10T15:52:10.859349Z 2022-01-10T15:52:11.524665Z 2022-01-10T15:52:10.653155Z 2022-01-10T15:52:10.879667Z 2022-01-10T15:52:10.292257Z 2022-01-10T15:52:11.099065Z 2022-01-10T15:52:10.224815Z 2022-01-10T15:52:10.896830Z 2022-01-10T15:58:49.842491Z 2022-01-10T15:58:49.703197Z 2022-01-10T15:58:49.570346Z 2022-01-10T15:58:49.352423Z 2022-01-10T15:58:49.136329Z 2022-01-10T15:58:29.863758Z 2022-01-10T15:58:29.511039Z 2022-01-10T15:58:29.266558Z 2022-01-10T15:58:28.893022Z 2022-01-10T15:58:28.583319Z 2022-01-10T15:58:28.291287Z 2022-01-10T15:58:27.855429Z 2022-01-10T15:58:27.478318Z 2022-01-10T15:58:27.323676Z 2022-01-10T15:58:26.955881Z 2022-01-10T15:58:26.981766Z 2022-01-10T15:58:26.286029Z 2022-01-10T15:58:26.034552Z 2022-01-10T15:58:25.937049Z 2022-01-10T15:58:25.580565Z 2022-01-10T15:58:25.817054Z 2022-01-10T15:58:25.339729Z 2022-01-10T15:58:24.901992Z 2022-01-10T15:58:24.500426Z 2022-01-10T15:58:24.067298Z 2022-01-10T15:58:24.167369Z 2022-01-10T15:58:23.200849Z 2022-01-10T15:58:23.104021Z 2022-01-10T15:58:22.662806Z 2022-01-10T15:58:22.582686Z 2022-01-10T15:58:22.296463Z 2022-01-10T15:58:22.146741Z 2022-01-10T15:58:21.591347Z 2022-01-10T15:58:21.067209Z 2022-01-10T15:58:20.898559Z 2022-01-10T15:58:20.653795Z 2022-01-10T15:58:20.031628Z 2022-01-10T15:58:20.241873Z 2022-01-10T15:58:19.517268Z 2022-01-10T15:58:19.358331Z 2022-01-10T15:58:18.931364Z 2022-01-10T15:58:18.562483Z 2022-01-10T15:58:18.380630Z 2022-01-10T15:58:17.901577Z 2022-01-10T15:58:17.644542Z 2022-01-10T15:58:17.652931Z 2022-01-10T15:58:16.856011Z 2022-01-10T15:58:17.042904Z 2022-01-10T15:58:16.499890Z 2022-01-10T15:58:15.924433Z 2022-01-10T15:58:15.769385Z 2022-01-10T15:58:15.519191Z 2022-01-10T15:58:15.505857Z 2022-01-10T15:58:15.429625Z 2022-01-10T15:58:14.884427Z 2022-01-10T15:58:14.609304Z 2022-01-10T15:58:14.785837Z 2022-01-10T15:58:14.189106Z 2022-01-10T15:58:13.830738Z 2022-01-10T15:58:13.430819Z 2022-01-10T15:58:13.253692Z 2022-01-10T15:58:12.665726Z 2022-01-10T15:58:12.343538Z 2022-01-10T15:58:12.296550Z 2022-01-10T15:58:11.782678Z 2022-01-10T15:58:11.531454Z 2022-01-10T15:58:11.243872Z 2022-01-10T15:58:10.689644Z 2022-01-10T15:58:10.529138Z 2022-01-10T15:58:10.241560Z 2022-01-10T15:58:10.268687Z 2022-01-10T15:58:09.901928Z 2022-01-10T15:58:09.691381Z 2022-01-10T15:58:09.312598Z 2022-01-10T15:58:09.179262Z 2022-01-10T15:58:08.852465Z 2022-01-10T15:58:08.616549Z 2022-01-10T15:58:01.560413Z 2022-01-10T15:58:00.856373Z 2022-01-10T15:58:00.551152Z 2022-01-10T15:57:59.966653Z 2022-01-10T15:57:59.363681Z 2022-01-10T15:57:58.860418Z 2022-01-10T15:57:57.895561Z 2022-01-10T15:57:56.582717Z 2022-01-10T15:57:55.970668Z 2022-01-10T15:57:54.739076Z 2022-01-10T15:57:54.119528Z 2022-01-10T15:57:53.819966Z 2022-01-10T15:57:52.976970Z 2022-01-10T15:57:52.679510Z 2022-01-10T15:57:52.216307Z 2022-01-10T15:57:46.610800Z 2022-01-10T15:57:46.319162Z 2022-01-10T15:57:45.843972Z 2022-01-10T15:57:45.827199Z 2022-01-10T15:57:44.540123Z 2022-01-10T15:57:44.437776Z 2022-01-10T15:57:44.083958Z 2022-01-10T15:57:43.995685Z 2022-01-10T16:02:49.556549Z 2022-01-10T16:02:49.224176Z 2022-01-10T16:02:49.113693Z 2022-01-10T16:02:39.760458Z 2022-01-10T16:02:48.807533Z 2022-01-10T16:02:39.133732Z 2022-01-10T16:02:39.042753Z 2022-01-10T16:02:38.497409Z 2022-01-10T16:02:38.256945Z 2022-01-10T16:02:37.852536Z 2022-01-10T16:02:37.438317Z 2022-01-10T16:02:37.022521Z 2022-01-10T16:02:36.824682Z 2022-01-10T16:02:36.021909Z 2022-01-10T16:02:35.802315Z 2022-01-10T16:02:35.148676Z 2022-01-10T16:02:35.024370Z 2022-01-10T16:02:34.691571Z 2022-01-10T16:02:34.452795Z 2022-01-10T16:02:34.073459Z 2022-01-10T16:02:34.134224Z 2022-01-10T16:02:33.467769Z 2022-01-10T16:02:33.480059Z 2022-01-10T16:02:32.956437Z 2022-01-10T16:02:32.877270Z 2022-01-10T16:02:32.744567Z 2022-01-10T16:02:32.392799Z 2022-01-10T16:02:31.769616Z 2022-01-10T16:02:31.544313Z 2022-01-10T16:02:31.060566Z 2022-01-10T16:02:30.736327Z 2022-01-10T16:02:30.448614Z 2022-01-10T16:02:30.269231Z 2022-01-10T16:02:29.929498Z 2022-01-10T16:02:29.487552Z 2022-01-10T16:02:29.168248Z 2022-01-10T16:02:29.082731Z 2022-01-10T16:02:28.239579Z 2022-01-10T16:02:27.830060Z 2022-01-10T16:02:27.653561Z 2022-01-10T16:02:27.312433Z 2022-01-10T16:02:27.330317Z 2022-01-10T16:02:26.756140Z 2022-01-10T16:02:26.281605Z 2022-01-10T16:02:26.068580Z 2022-01-10T16:02:25.681834Z 2022-01-10T16:02:25.403753Z 2022-01-10T16:02:24.911187Z 2022-01-10T16:02:24.827938Z 2022-01-10T16:02:24.586245Z 2022-01-10T16:02:24.055616Z 2022-01-10T16:02:23.608070Z 2022-01-10T16:02:23.148118Z 2022-01-10T16:02:22.743659Z 2022-01-10T16:02:22.583774Z 2022-01-10T16:02:22.147110Z 2022-01-10T16:02:21.726555Z 2022-01-10T16:02:21.065610Z 2022-01-10T16:02:21.085945Z 2022-01-10T16:02:20.243201Z 2022-01-10T16:02:20.074915Z 2022-01-10T16:02:19.746960Z 2022-01-10T16:02:19.409161Z 2022-01-10T16:02:19.072042Z 2022-01-10T16:02:18.825846Z 2022-01-10T16:02:18.480810Z 2022-01-10T16:02:18.239854Z 2022-01-10T16:02:17.930440Z 2022-01-10T16:02:17.488757Z 2022-01-10T16:02:17.302699Z 2022-01-10T16:02:16.829768Z 2022-01-10T16:02:16.471123Z 2022-01-10T16:02:16.316140Z 2022-01-10T16:02:15.836477Z 2022-01-10T16:02:15.813044Z 2022-01-10T16:02:15.571667Z 2022-01-10T16:02:15.378457Z 2022-01-10T16:02:15.315651Z 2022-01-10T16:02:15.078919Z 2022-01-10T16:02:05.742515Z 2022-01-10T16:02:05.217350Z 2022-01-10T16:02:14.810561Z 2022-01-10T16:02:04.967348Z 2022-01-10T16:02:04.857793Z 2022-01-10T16:02:03.813546Z 2022-01-10T16:02:04.669820Z 2022-01-10T16:02:03.305116Z 2022-01-10T16:02:03.634413Z 2022-01-10T16:02:01.822052Z 2022-01-10T16:02:01.637875Z 2022-01-10T16:02:00.840446Z 2022-01-10T16:02:00.808626Z 2022-01-10T16:02:00.382451Z 2022-01-10T16:02:00.260341Z 2022-01-10T16:01:55.317355Z 2022-01-10T16:01:55.344995Z 2022-01-10T16:01:54.767379Z 2022-01-10T16:01:53.879655Z 2022-01-10T16:01:53.317061Z 2022-01-10T16:01:53.241958Z 2022-01-10T16:06:56.453319Z 2022-01-10T16:06:56.237314Z 2022-01-10T16:06:55.908949Z 2022-01-10T16:06:55.657539Z 2022-01-10T16:06:55.735850Z 2022-01-10T16:06:55.162477Z 2022-01-10T16:06:54.679150Z 2022-01-10T16:06:54.878103Z 2022-01-10T16:06:54.101915Z 2022-01-10T16:06:53.750425Z 2022-01-10T16:06:53.320553Z 2022-01-10T16:06:52.791194Z 2022-01-10T16:06:52.581591Z 2022-01-10T16:06:52.364420Z 2022-01-10T16:06:52.142443Z 2022-01-10T16:06:51.506313Z 2022-01-10T16:06:51.262787Z 2022-01-10T16:06:50.846037Z 2022-01-10T16:06:50.492862Z 2022-01-10T16:06:50.424547Z 2022-01-10T16:06:49.928536Z 2022-01-10T16:06:49.770460Z 2022-01-10T16:06:49.302563Z 2022-01-10T16:06:49.132959Z 2022-01-10T16:06:48.675358Z 2022-01-10T16:06:48.116070Z 2022-01-10T16:06:47.637144Z 2022-01-10T16:06:47.247469Z 2022-01-10T16:06:46.937386Z 2022-01-10T16:06:46.553299Z 2022-01-10T16:06:46.425956Z 2022-01-10T16:06:46.197879Z 2022-01-10T16:06:45.857891Z 2022-01-10T16:06:45.558432Z 2022-01-10T16:06:45.289739Z 2022-01-10T16:06:45.291834Z 2022-01-10T16:06:44.540674Z 2022-01-10T16:06:44.505337Z 2022-01-10T16:06:43.355559Z 2022-01-10T16:06:43.226269Z 2022-01-10T16:06:42.439902Z 2022-01-10T16:06:42.128888Z 2022-01-10T16:06:41.912762Z 2022-01-10T16:06:41.686287Z 2022-01-10T16:06:41.521937Z 2022-01-10T16:06:40.914531Z 2022-01-10T16:06:40.459680Z 2022-01-10T16:06:40.021161Z 2022-01-10T16:06:39.973936Z 2022-01-10T16:06:39.087680Z 2022-01-10T16:06:38.666937Z 2022-01-10T16:06:38.575956Z 2022-01-10T16:06:37.588253Z 2022-01-10T16:06:37.369690Z 2022-01-10T16:06:37.228672Z 2022-01-10T16:06:36.888251Z 2022-01-10T16:06:36.514636Z 2022-01-10T16:06:36.626326Z 2022-01-10T16:06:36.035089Z 2022-01-10T16:06:35.638797Z 2022-01-10T16:06:35.334665Z 2022-01-10T16:06:34.882330Z 2022-01-10T16:06:34.599389Z 2022-01-10T16:06:34.283909Z 2022-01-10T16:06:33.870151Z 2022-01-10T16:06:33.788993Z 2022-01-10T16:06:33.374529Z 2022-01-10T16:06:33.378724Z 2022-01-10T16:06:32.944577Z 2022-01-10T16:06:32.667831Z 2022-01-10T16:06:32.488655Z 2022-01-10T16:06:32.045124Z 2022-01-10T16:06:24.174183Z 2022-01-10T16:06:24.035268Z 2022-01-10T16:06:23.448280Z 2022-01-10T16:06:22.996056Z 2022-01-10T16:06:22.758312Z 2022-01-10T16:06:22.490291Z 2022-01-10T16:06:22.186279Z 2022-01-10T16:06:21.860289Z 2022-01-10T16:06:21.786461Z 2022-01-10T16:06:21.298357Z 2022-01-10T16:06:21.073947Z 2022-01-10T16:06:20.874665Z 2022-01-10T16:06:20.422883Z 2022-01-10T16:06:20.275634Z 2022-01-10T16:06:20.048195Z 2022-01-10T16:06:19.733447Z 2022-01-10T16:06:19.686193Z 2022-01-10T16:06:19.262224Z 2022-01-10T16:06:19.083300Z 2022-01-10T16:06:18.752111Z 2022-01-10T16:06:18.407513Z 2022-01-10T16:06:18.280666Z 2022-01-10T16:06:18.094478Z 2022-01-10T16:06:10.555157Z 2022-01-10T16:06:10.411852Z 2022-01-10T16:06:10.045252Z 2022-01-10T16:06:09.630296Z 2022-01-10T16:06:09.099564Z 2022-01-10T16:11:34.373664Z 2022-01-10T16:11:34.136328Z 2022-01-10T16:11:33.767450Z 2022-01-10T16:11:33.255274Z 2022-01-10T16:11:33.051757Z 2022-01-10T16:11:32.676618Z 2022-01-10T16:11:32.726011Z 2022-01-10T16:11:32.395152Z 2022-01-10T16:11:32.365690Z 2022-01-10T16:11:31.813340Z 2022-01-10T16:11:31.544242Z 2022-01-10T16:11:31.470593Z 2022-01-10T16:11:31.097667Z 2022-01-10T16:11:30.685299Z 2022-01-10T16:11:29.919945Z 2022-01-10T16:11:29.375071Z 2022-01-10T16:11:29.221626Z 2022-01-10T16:11:28.665951Z 2022-01-10T16:11:28.355808Z 2022-01-10T16:11:28.512131Z 2022-01-10T16:11:27.794538Z 2022-01-10T16:11:27.680131Z 2022-01-10T16:11:27.072791Z 2022-01-10T16:11:26.872379Z 2022-01-10T16:11:26.515671Z 2022-01-10T16:11:26.117013Z 2022-01-10T16:11:25.770667Z 2022-01-10T16:11:25.298264Z 2022-01-10T16:11:24.326021Z 2022-01-10T16:11:24.372073Z 2022-01-10T16:11:23.820195Z 2022-01-10T16:11:23.267273Z 2022-01-10T16:11:23.019038Z 2022-01-10T16:11:22.081819Z 2022-01-10T16:11:21.621842Z 2022-01-10T16:11:21.147398Z 2022-01-10T16:11:20.725893Z 2022-01-10T16:11:20.227503Z 2022-01-10T16:11:19.689499Z 2022-01-10T16:11:19.151298Z 2022-01-10T16:11:18.611488Z 2022-01-10T16:11:17.941858Z 2022-01-10T16:10:54.150886Z 2022-01-10T16:11:17.286498Z 2022-01-10T16:11:16.971775Z 2022-01-10T16:11:16.867110Z 2022-01-10T16:10:53.997470Z 2022-01-10T16:11:16.416483Z 2022-01-10T16:11:16.319589Z 2022-01-10T16:10:53.341972Z 2022-01-10T16:10:52.970805Z 2022-01-10T16:10:52.290608Z 2022-01-10T16:10:52.285962Z 2022-01-10T16:10:51.533580Z 2022-01-10T16:10:50.993860Z 2022-01-10T16:10:50.680725Z 2022-01-10T16:10:50.535173Z 2022-01-10T16:10:50.191690Z 2022-01-10T16:10:50.047140Z 2022-01-10T16:10:49.852955Z 2022-01-10T16:10:49.338374Z 2022-01-10T16:10:49.302808Z 2022-01-10T16:10:48.684441Z 2022-01-10T16:10:48.074572Z 2022-01-10T16:10:47.986630Z 2022-01-10T16:10:47.644207Z 2022-01-10T16:10:47.071703Z 2022-01-10T16:10:46.737442Z 2022-01-10T16:10:46.571363Z 2022-01-10T16:10:46.255700Z 2022-01-10T16:10:45.809845Z 2022-01-10T16:10:45.402996Z 2022-01-10T16:10:45.263093Z 2022-01-10T16:10:45.297965Z 2022-01-10T16:10:44.910181Z 2022-01-10T16:10:44.683768Z 2022-01-10T16:10:44.414230Z 2022-01-10T16:10:43.923522Z 2022-01-10T16:10:43.555062Z 2022-01-10T16:10:43.149663Z 2022-01-10T16:10:42.768538Z 2022-01-10T16:10:42.562461Z 2022-01-10T16:10:42.287488Z 2022-01-10T16:10:42.124049Z 2022-01-10T16:10:41.907727Z 2022-01-10T16:10:41.750829Z 2022-01-10T16:10:41.341896Z 2022-01-10T16:10:41.127383Z 2022-01-10T16:10:41.192229Z 2022-01-10T16:10:40.869705Z 2022-01-10T16:10:40.845257Z 2022-01-10T16:10:33.317800Z 2022-01-10T16:10:32.401287Z 2022-01-10T16:10:32.379069Z 2022-01-10T16:10:32.696348Z 2022-01-10T16:10:31.616996Z 2022-01-10T16:10:30.951002Z 2022-01-10T16:10:30.496472Z 2022-01-10T16:10:27.643687Z 2022-01-10T16:10:27.552557Z", + "started_diff": 54.737813800000005, + "successful_job_ids": " [99, 98, 97, 95, 94, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 78, 77, 76, 75, 74, 73, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 176, 175, 174, 173, 172, 171, 170, 169, 168, 166, 165, 164, 162, 159, 157, 156, 153, 152, 151, 150, 149, 148, 147, 146, 145, 143, 142, 141, 139, 138, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 105, 103, 102, 100, 276, 275, 274, 273, 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 376, 375, 374, 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 476, 475, 474, 473, 472, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 576, 575, 574, 573, 572, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477]" + }, + "started": "2022-01-10T16:37:32.121931+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T03_04_05_928_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T03_04_05_928_0000-chatty.json new file mode 100644 index 00000000..4fdbaba5 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T03_04_05_928_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-11T03_04_05_928_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-11T04:51:54.533721+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-api.json new file mode 100644 index 00000000..d14df280 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-api.json @@ -0,0 +1,1244 @@ +{ + "ended": "2022-01-11T11:58:16.570754+00:00", + "id": "run-2022-01-11T08_57_55_678_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2369.5, + "max": 24519.933333333334, + "mean": 5308.735584506569, + "median": 3264.6666666666665, + "min": 1348.2666666666667, + "non_zero_mean": 5308.735584506569, + "non_zero_median": 3264.6666666666665, + "percentile25": 2703.2333333333336, + "percentile75": 5072.733333333334, + "percentile90": 12195.586666666668, + "percentile99": 23147.567999999992, + "percentile999": 24432.60000000002, + "range": 23171.666666666668, + "samples": 263, + "stdev": 4774.108533759505, + "sum": 1396197.4587252284 + } + }, + "cpu": { + "system": { + "iqr": 0.18600088900742268, + "max": 1.7486666666666681, + "mean": 0.2768998557683042, + "median": 0.11066666666666455, + "min": 0.03133333333333137, + "non_zero_mean": 0.2768998557683042, + "non_zero_median": 0.11066666666666455, + "percentile25": 0.0613333333333344, + "percentile75": 0.24733422234075708, + "percentile90": 0.9008000000000009, + "percentile99": 1.6572799999999972, + "percentile999": 1.740108000000004, + "range": 1.7173333333333367, + "samples": 263, + "stdev": 0.39276209648987026, + "sum": 72.82466206706401 + }, + "user": { + "iqr": 0.5616666666666561, + "max": 6.213333333333321, + "mean": 1.3653181341111098, + "median": 0.7539999999999965, + "min": 0.3860000000000279, + "non_zero_mean": 1.3653181341111098, + "non_zero_median": 0.7539999999999965, + "percentile25": 0.5836666666667043, + "percentile75": 1.1453333333333604, + "percentile90": 3.8290666666666797, + "percentile99": 6.163386666666683, + "percentile999": 6.204250666666665, + "range": 5.827333333333294, + "samples": 263, + "stdev": 1.461990290969789, + "sum": 359.078669271222 + } + }, + "entropy_available_bits": { + "iqr": 1.7999999999999998, + "max": 215.6, + "mean": 20.136375139379826, + "median": 1.7333333333333334, + "min": 0.13333333333333333, + "non_zero_mean": 20.136375139379826, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.7333333333333333, + "percentile75": 2.533333333333333, + "percentile90": 5.333333333333341, + "percentile99": 214.41866666666664, + "percentile999": 215.35546666666673, + "range": 215.46666666666667, + "samples": 263, + "stdev": 56.575247221825514, + "sum": 5295.866661656895 + }, + "memory": { + "used": { + "iqr": 366243840.0, + "max": 15527211008.0, + "mean": 6520117832.030418, + "median": 6079807488.0, + "min": 5642035200.0, + "non_zero_mean": 6520117832.030418, + "non_zero_median": 6079807488.0, + "percentile25": 5955719168.0, + "percentile75": 6321963008.0, + "percentile90": 7093335654.4, + "percentile99": 13275142143.999996, + "percentile999": 15436020989.952019, + "range": 9885175808.0, + "samples": 263, + "stdev": 1449436236.356484, + "sum": 1714790989824.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 28.2, + "mean": 0.13891001267427122, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6.088888888888889, + "non_zero_median": 1.6666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.650666666666666, + "percentile999": 21.737333333334725, + "range": 28.2, + "samples": 263, + "stdev": 1.7583059146509403, + "sum": 36.53333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 84.9, + "max": 1622.3333333333335, + "mean": 123.41368145853812, + "median": 28.066666666666663, + "min": 0.0, + "non_zero_mean": 126.29493472216159, + "non_zero_median": 31.73333333333333, + "percentile25": 1.5666666666666667, + "percentile75": 86.46666666666667, + "percentile90": 399.8266666666667, + "percentile99": 1197.1999999999998, + "percentile999": 1527.6814666666871, + "range": 1622.3333333333335, + "samples": 263, + "stdev": 255.25092207070213, + "sum": 32457.798223595528 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 926.9666666666667, + "max": 3256.866666666667, + "mean": 1619.0020278833967, + "median": 1488.2, + "min": 510.53333333333336, + "non_zero_mean": 1619.0020278833967, + "non_zero_median": 1488.2, + "percentile25": 1160.1666666666665, + "percentile75": 2087.133333333333, + "percentile90": 2451.2000000000003, + "percentile99": 2935.494666666666, + "percentile999": 3194.1264000000137, + "range": 2746.3333333333335, + "samples": 263, + "stdev": 590.3756736979095, + "sum": 425797.5333333337 + } + }, + "cpu": { + "system": { + "iqr": 0.042333333333333674, + "max": 0.25733333333333425, + "mean": 0.08456273764258555, + "median": 0.07200000000000273, + "min": 0.01599999999999871, + "non_zero_mean": 0.08456273764258555, + "non_zero_median": 0.07200000000000273, + "percentile25": 0.056333333333332784, + "percentile75": 0.09866666666666646, + "percentile90": 0.15626666666666666, + "percentile99": 0.2135866666666664, + "percentile999": 0.25488800000000067, + "range": 0.24133333333333554, + "samples": 263, + "stdev": 0.04403407979991525, + "sum": 22.23999999999998 + }, + "user": { + "iqr": 0.12299999999999991, + "max": 0.9986666666666679, + "mean": 0.20257794676806085, + "median": 0.19333333333332803, + "min": 0.020000000000000757, + "non_zero_mean": 0.20257794676806085, + "non_zero_median": 0.19333333333332803, + "percentile25": 0.13300000000000126, + "percentile75": 0.25600000000000117, + "percentile90": 0.33146666666666386, + "percentile99": 0.48851999999999857, + "percentile999": 0.8872293333333576, + "range": 0.9786666666666671, + "samples": 263, + "stdev": 0.10692757802281007, + "sum": 53.27800000000001 + } + }, + "entropy_available_bits": { + "iqr": 1.4333333333333331, + "max": 213.2, + "mean": 11.246894803548797, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 21.43429951690821, + "non_zero_median": 1.4, + "percentile25": 0.0, + "percentile75": 1.4333333333333331, + "percentile90": 3.1066666666666682, + "percentile99": 212.38400000000001, + "percentile999": 213.1476, + "range": 213.2, + "samples": 263, + "stdev": 45.842512588455854, + "sum": 2957.9333333333348 + }, + "memory": { + "used": { + "iqr": 21817344.0, + "max": 1054638080.0, + "mean": 424552580.38022816, + "median": 384884736.0, + "min": 362844160.0, + "non_zero_mean": 424552580.38022816, + "non_zero_median": 384884736.0, + "percentile25": 376856576.0, + "percentile75": 398673920.0, + "percentile90": 536073830.40000004, + "percentile99": 997699010.5599998, + "percentile999": 1046724657.1520017, + "range": 691793920.0, + "samples": 263, + "stdev": 121268501.80720747, + "sum": 111657328640.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6553.6, + "max": 59774898.06666667, + "mean": 1131032.8646387833, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3004663.064646465, + "non_zero_median": 11468.8, + "percentile25": 0.0, + "percentile75": 6553.6, + "percentile90": 35061.760000000024, + "percentile99": 58461895.229333326, + "percentile999": 59755008.98293334, + "range": 59774898.06666667, + "samples": 263, + "stdev": 8058141.394844132, + "sum": 297461643.4 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 411.381349, + "mean": 2.2028850646387834, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 25.189511826086957, + "non_zero_median": 0.088093, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.3810726599999979, + "percentile999": 347.01189498201387, + "range": 411.381349, + "samples": 263, + "stdev": 27.310367995828344, + "sum": 579.3587719999999 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.24315900000000001, + "max": 426.371198, + "mean": 25.872867653992394, + "median": 0.0, + "min": -1.6e-05, + "non_zero_mean": 58.158668316239314, + "non_zero_median": 0.699581, + "percentile25": 0.0, + "percentile75": 0.24315900000000001, + "percentile90": 88.84515320000011, + "percentile99": 372.0754209399999, + "percentile999": 418.5113046340017, + "range": 426.371214, + "samples": 263, + "stdev": 77.1141095855917, + "sum": 6804.564193000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 5.0, + "mean": 0.12927756653992395, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.6153846153846154, + "non_zero_median": 2.3333333333333335, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.5679999999999987, + "percentile999": 4.720533333333393, + "range": 5.0, + "samples": 263, + "stdev": 0.6174316363343862, + "sum": 34.00000000000001 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9997.066666666668, + "mean": 429.0750316856781, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8680.517948717948, + "non_zero_median": 8806.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9976.013333333334, + "percentile999": 9994.830933333335, + "range": 9997.066666666668, + "samples": 263, + "stdev": 1906.3635913184698, + "sum": 112846.73333333334 + }, + "pg_stat_database_blks_hit": { + "iqr": 21282.2, + "max": 198043.93333333332, + "mean": 25667.95969581749, + "median": 25946.333333333332, + "min": 2268.5333333333333, + "non_zero_mean": 25667.95969581749, + "non_zero_median": 25946.333333333332, + "percentile25": 12165.7, + "percentile75": 33447.9, + "percentile90": 43374.66666666667, + "percentile99": 65154.99733333321, + "percentile999": 189767.99960000176, + "range": 195775.4, + "samples": 263, + "stdev": 20143.2966317538, + "sum": 6750673.4 + }, + "pg_stat_database_blks_read": { + "iqr": 0.6, + "max": 214.53333333333333, + "mean": 1.420278833967047, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.770300157977883, + "non_zero_median": 0.4, + "percentile25": 0.06666666666666667, + "percentile75": 0.6666666666666666, + "percentile90": 1.973333333333335, + "percentile99": 4.443999999999998, + "percentile999": 160.03733333334506, + "range": 214.53333333333333, + "samples": 263, + "stdev": 13.226492848396713, + "sum": 373.5333333333329 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 263, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.6, + "max": 214.53333333333333, + "mean": 1.420278833967047, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.770300157977883, + "non_zero_median": 0.4, + "percentile25": 0.06666666666666667, + "percentile75": 0.6666666666666666, + "percentile90": 1.973333333333335, + "percentile99": 4.443999999999998, + "percentile999": 160.03733333334506, + "range": 214.53333333333333, + "samples": 263, + "stdev": 13.226492848396713, + "sum": 373.5333333333329 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.333333333333333, + "max": 4005.733333333333, + "mean": 41.27198986058301, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 103.37650793650793, + "non_zero_median": 15.266666666666667, + "percentile25": 0.0, + "percentile75": 2.333333333333333, + "percentile90": 59.440000000000005, + "percentile99": 85.60533333333328, + "percentile999": 4004.6853333333333, + "range": 4005.733333333333, + "samples": 263, + "stdev": 348.2522138722028, + "sum": 10854.533333333338 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11572.166666666666, + "max": 460614.73333333334, + "mean": 14962.277059569074, + "median": 13789.066666666668, + "min": 1297.6, + "non_zero_mean": 14962.277059569074, + "non_zero_median": 13789.066666666668, + "percentile25": 5997.333333333334, + "percentile75": 17569.5, + "percentile90": 23688.880000000005, + "percentile99": 37362.10933333332, + "percentile999": 353085.6889333565, + "range": 459317.13333333336, + "samples": 263, + "stdev": 28907.421722056155, + "sum": 3935078.866666669 + }, + "pg_stat_database_tup_inserted": { + "iqr": 8.7, + "max": 8000.2, + "mean": 46.165272496831435, + "median": 2.6, + "min": 0.0, + "non_zero_mean": 54.938763197586724, + "non_zero_median": 3.7333333333333334, + "percentile25": 1.3333333333333333, + "percentile75": 10.033333333333333, + "percentile90": 55.86666666666668, + "percentile99": 146.36133333333333, + "percentile999": 5946.364533333775, + "range": 8000.2, + "samples": 263, + "stdev": 493.47338215062683, + "sum": 12141.466666666662 + }, + "pg_stat_database_tup_returned": { + "iqr": 35627.166666666664, + "max": 526220.0, + "mean": 45130.85424588086, + "median": 37547.8, + "min": 3149.266666666667, + "non_zero_mean": 45130.85424588086, + "non_zero_median": 37547.8, + "percentile25": 19412.933333333334, + "percentile75": 55040.1, + "percentile90": 90393.68, + "percentile99": 173003.13599999997, + "percentile999": 515963.39866666886, + "range": 523070.73333333334, + "samples": 263, + "stdev": 52177.49360732776, + "sum": 11869414.666666672 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.933333333333332, + "max": 61.4, + "mean": 10.351330798479088, + "median": 6.866666666666666, + "min": 0.13333333333333333, + "non_zero_mean": 10.351330798479088, + "non_zero_median": 6.866666666666666, + "percentile25": 3.4000000000000004, + "percentile75": 16.333333333333332, + "percentile90": 22.813333333333336, + "percentile99": 40.23599999999996, + "percentile999": 59.79306666666701, + "range": 61.266666666666666, + "samples": 263, + "stdev": 9.717322598594247, + "sum": 2722.4000000000005 + }, + "pg_stat_database_xact_commit": { + "iqr": 126.56666666666666, + "max": 721.3333333333334, + "mean": 122.63472750316856, + "median": 89.73333333333333, + "min": 5.066666666666666, + "non_zero_mean": 122.63472750316856, + "non_zero_median": 89.73333333333333, + "percentile25": 39.46666666666667, + "percentile75": 166.03333333333333, + "percentile90": 266.29333333333335, + "percentile99": 562.9119999999999, + "percentile999": 716.0933333333345, + "range": 716.2666666666667, + "samples": 263, + "stdev": 124.67826344090297, + "sum": 32252.933333333345 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.12648922686945502, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.12648922686945502, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5586666666666663, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 263, + "stdev": 0.14551410057806102, + "sum": 33.26666666666667 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 263, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 23.866666666666664, + "max": 147.60000000000002, + "mean": 39.08441064638783, + "median": 31.666666666666664, + "min": 2.8, + "non_zero_mean": 39.08441064638783, + "non_zero_median": 31.666666666666664, + "percentile25": 23.2, + "percentile75": 47.06666666666666, + "percentile90": 72.66666666666667, + "percentile99": 111.52266666666662, + "percentile999": 140.19413333333495, + "range": 144.8, + "samples": 263, + "stdev": 23.13515086568422, + "sum": 10279.2 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2022-01-11T10:52:31.003828+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "PASS", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2022-01-11T08_57_55_678_0000", + "ended": "2022-01-11T11:58:11.433835+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.017823353250605578, + "iterations": 1, + "max": 0.5596723179996843, + "mean": 0.5442409213332212, + "median": 0.5429783820000011, + "min": 0.5241682349997063, + "rounds": 15, + "stddev": 0.011309798432395365 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.007562134499949025, + "iterations": 1, + "max": 0.5526895960001639, + "mean": 0.5396342008000526, + "median": 0.5413831619998746, + "min": 0.5258900080007152, + "rounds": 15, + "stddev": 0.0075744309153849354 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.014684437999676447, + "iterations": 1, + "max": 0.5810878570000568, + "mean": 0.41533960579999984, + "median": 0.39850186049989134, + "min": 0.387971989000107, + "rounds": 10, + "stddev": 0.05871493911879718 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.006970508999529557, + "iterations": 1, + "max": 0.3519583520001106, + "mean": 0.25954496513000774, + "median": 0.2581625415000417, + "min": 0.2436642890006624, + "rounds": 100, + "stddev": 0.011524173886426847 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.012016478500299854, + "iterations": 1, + "max": 0.5689027309999801, + "mean": 0.47106980962998934, + "median": 0.46910466349982016, + "min": 0.44958714099993813, + "rounds": 100, + "stddev": 0.014383736696583263 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.12288666199947329, + "iterations": 1, + "max": 1.5384094119999645, + "mean": 1.3522033465997993, + "median": 1.3403718834997562, + "min": 1.2669507359996715, + "rounds": 10, + "stddev": 0.08789510185165897 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0028313369994066306, + "iterations": 1, + "max": 0.07804784600011772, + "mean": 0.07453092792859804, + "median": 0.07393210000009276, + "min": 0.07206464099999721, + "rounds": 14, + "stddev": 0.0018090606314850723 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.001778519000254164, + "iterations": 1, + "max": 0.07489938300022914, + "mean": 0.07171081607144256, + "median": 0.07158448100017267, + "min": 0.06931252399954246, + "rounds": 14, + "stddev": 0.0015100433054480095 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.007463003999873763, + "iterations": 1, + "max": 0.11575932100004138, + "mean": 0.10238106889992196, + "median": 0.10031766900010552, + "min": 0.09686937799961015, + "rounds": 10, + "stddev": 0.005858887631439437 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.007990578000317328, + "iterations": 1, + "max": 0.120785364000767, + "mean": 0.11253329600031066, + "median": 0.1138351204999708, + "min": 0.10156080600063433, + "rounds": 10, + "stddev": 0.006245060923456575 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.004762561999996251, + "iterations": 1, + "max": 0.38938677399983135, + "mean": 0.29076273849987044, + "median": 0.2799219399998947, + "min": 0.2724031660000037, + "rounds": 10, + "stddev": 0.03493974677688464 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.0037552089988821535, + "iterations": 1, + "max": 0.2120421620002162, + "mean": 0.12005191759999434, + "median": 0.11052374049950231, + "min": 0.10532652300025802, + "rounds": 10, + "stddev": 0.03243624952630942 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0034763690000545466, + "iterations": 1, + "max": 0.07688442899961956, + "mean": 0.07310399635720646, + "median": 0.07354741449989888, + "min": 0.06856727600006707, + "rounds": 14, + "stddev": 0.0024502375930434027 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.028560636000293016, + "iterations": 1, + "max": 0.7089736959997026, + "mean": 0.6276611319999574, + "median": 0.6181369695000285, + "min": 0.5919984040001509, + "rounds": 10, + "stddev": 0.03246585074285473 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0031492487505602185, + "iterations": 1, + "max": 0.09998783499941055, + "mean": 0.09713240818175771, + "median": 0.0970330219997777, + "min": 0.09376548000000184, + "rounds": 11, + "stddev": 0.0019572530890495155 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.003750902000319911, + "iterations": 1, + "max": 0.10982455799967283, + "mean": 0.10661635859987655, + "median": 0.10716083349961991, + "min": 0.10250005499983672, + "rounds": 10, + "stddev": 0.0022769069239365804 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.007045153000035498, + "iterations": 1, + "max": 0.18212270100048045, + "mean": 0.17569539390015051, + "median": 0.17613271900017935, + "min": 0.1708715509994363, + "rounds": 10, + "stddev": 0.004311946393823417 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.0028813289991376223, + "iterations": 1, + "max": 0.07741603100021166, + "mean": 0.07459286192858729, + "median": 0.07417572849999488, + "min": 0.07131060599931516, + "rounds": 14, + "stddev": 0.0018430391636613066 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.002692908250310211, + "iterations": 1, + "max": 0.07759208400057105, + "mean": 0.07493156692320414, + "median": 0.07573374200001126, + "min": 0.06858487099998456, + "rounds": 13, + "stddev": 0.002352857323824358 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.019811633999779588, + "iterations": 1, + "max": 0.4169999369996731, + "mean": 0.3752380596999501, + "median": 0.3675053830002071, + "min": 0.3523123220002162, + "rounds": 10, + "stddev": 0.018431327077698484 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.0076966449996689335, + "iterations": 1, + "max": 0.3869333579996237, + "mean": 0.3786538407998705, + "median": 0.3792231829997945, + "min": 0.3707072099996367, + "rounds": 10, + "stddev": 0.004969749871429826 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.005841192999469058, + "iterations": 1, + "max": 0.10851820000061707, + "mean": 0.09549883160016179, + "median": 0.0952930534999723, + "min": 0.08983692899982998, + "rounds": 10, + "stddev": 0.005344769889468911 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.003483049499664048, + "iterations": 1, + "max": 0.10518662499998754, + "mean": 0.09000079825007863, + "median": 0.0881817084996328, + "min": 0.08450735600035841, + "rounds": 12, + "stddev": 0.006083562526674602 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.008295362000353634, + "iterations": 1, + "max": 0.18474003499977698, + "mean": 0.17931174559998908, + "median": 0.18071921750015463, + "min": 0.1697157739999966, + "rounds": 10, + "stddev": 0.005799175578186389 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.002210237749523003, + "iterations": 1, + "max": 0.07538899199971638, + "mean": 0.0725639148665626, + "median": 0.07205692299976363, + "min": 0.06956039699980465, + "rounds": 15, + "stddev": 0.0018302670035777863 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.006430966998777876, + "iterations": 1, + "max": 0.21566887999961182, + "mean": 0.20814715279984738, + "median": 0.20746666249988266, + "min": 0.20067458699941199, + "rounds": 10, + "stddev": 0.004754302231147222 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.002999973000441969, + "iterations": 1, + "max": 0.08218475200010289, + "mean": 0.07977243878576701, + "median": 0.07989904700025363, + "min": 0.07670410599985189, + "rounds": 14, + "stddev": 0.0017422825021440177 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.019105704999674344, + "iterations": 1, + "max": 0.36567140399984055, + "mean": 0.3496825646000616, + "median": 0.3480356140003096, + "min": 0.33191403599994374, + "rounds": 10, + "stddev": 0.011481256425010532 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.006347157000163861, + "iterations": 1, + "max": 0.24888140200073394, + "mean": 0.23023003899998, + "median": 0.2311818204998417, + "min": 0.21784400999968057, + "rounds": 10, + "stddev": 0.008053537787910797 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.003534816999490431, + "iterations": 1, + "max": 0.17153278600017074, + "mean": 0.16832899170012752, + "median": 0.16810974350028118, + "min": 0.163461799000288, + "rounds": 10, + "stddev": 0.0024553657034626273 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.12859126499915874, + "iterations": 1, + "max": 0.25973239799986914, + "mean": 0.17208893239994721, + "median": 0.16902469600017866, + "min": 0.10252337600013561, + "rounds": 10, + "stddev": 0.07018338772438076 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.004424047999236791, + "iterations": 1, + "max": 0.07887207600015245, + "mean": 0.07437464807123563, + "median": 0.0748356569997668, + "min": 0.07023244099946169, + "rounds": 14, + "stddev": 0.0026166028676519845 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.0031188297498374595, + "iterations": 1, + "max": 0.0999946549991364, + "mean": 0.0956475136363224, + "median": 0.09521827600019606, + "min": 0.09273613799996383, + "rounds": 11, + "stddev": 0.002164985162098475 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0028054670001438353, + "iterations": 1, + "max": 0.07685547199980647, + "mean": 0.07277219057141338, + "median": 0.07290507600009732, + "min": 0.06995747599921742, + "rounds": 14, + "stddev": 0.002028948839979616 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.006175968999741599, + "iterations": 1, + "max": 0.1408592999996472, + "mean": 0.1282081020000078, + "median": 0.1278852824993919, + "min": 0.12044091799998569, + "rounds": 10, + "stddev": 0.005553307925667591 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.00197757800015097, + "iterations": 1, + "max": 0.0870983820004767, + "mean": 0.07352199464269558, + "median": 0.07188176950012348, + "min": 0.0707697489997372, + "rounds": 14, + "stddev": 0.004231460812229434 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.028641904999858525, + "iterations": 1, + "max": 0.6131740990003891, + "mean": 0.5839047616000244, + "median": 0.5859606789999816, + "min": 0.5602555949999442, + "rounds": 10, + "stddev": 0.01695930169335228 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.032603065999865066, + "iterations": 1, + "max": 0.7534062390004692, + "mean": 0.6225323958000445, + "median": 0.6034557445000246, + "min": 0.5880408140001236, + "rounds": 10, + "stddev": 0.04887336675629257 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.005553136000344239, + "iterations": 1, + "max": 0.39253774900043936, + "mean": 0.2867896389001544, + "median": 0.2753447400004916, + "min": 0.2710449440000957, + "rounds": 10, + "stddev": 0.03729692467632429 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.002609611000480072, + "iterations": 1, + "max": 0.07679533000009542, + "mean": 0.07393432392875963, + "median": 0.07428828550018807, + "min": 0.07090460900053586, + "rounds": 14, + "stddev": 0.0017223024438759883 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0020203889998811064, + "iterations": 1, + "max": 0.0750494339999932, + "mean": 0.07212525692843233, + "median": 0.0721794464998311, + "min": 0.06858262999958242, + "rounds": 14, + "stddev": 0.0017713789273202773 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.003950442999666848, + "iterations": 1, + "max": 0.08130653999978676, + "mean": 0.07759824371428944, + "median": 0.07720805400003883, + "min": 0.07505463300003612, + "rounds": 14, + "stddev": 0.0020460556829737324 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.00335138900027232, + "iterations": 1, + "max": 0.087036631999581, + "mean": 0.07550303064291645, + "median": 0.07422932850022335, + "min": 0.07259371600048325, + "rounds": 14, + "stddev": 0.003627243330574389 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.010971890999826428, + "iterations": 1, + "max": 0.32340075699994486, + "mean": 0.3029169686999012, + "median": 0.3014646404999439, + "min": 0.2931998399999429, + "rounds": 10, + "stddev": 0.009651903698899981 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.008013265000045067, + "iterations": 1, + "max": 0.23680217200035258, + "mean": 0.22795954370021718, + "median": 0.22726663900039057, + "min": 0.2197583269999086, + "rounds": 10, + "stddev": 0.005806585992679614 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.01651519900042331, + "iterations": 1, + "max": 1.0515961680002874, + "mean": 0.9495882553999764, + "median": 0.940172263000477, + "min": 0.9271148569996512, + "rounds": 10, + "stddev": 0.036814953683800646 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.16833630000110134, + "iterations": 1, + "max": 9.229790645999856, + "mean": 8.813222646899977, + "median": 8.741473840000253, + "min": 8.654367801999797, + "rounds": 10, + "stddev": 0.18820929459840904 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.020378506999804813, + "iterations": 1, + "max": 1.0736694580000403, + "mean": 1.0292958622000696, + "median": 1.0298231085002953, + "min": 0.9808272090003811, + "rounds": 10, + "stddev": 0.025614112204502302 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.11536765100026969, + "iterations": 1, + "max": 9.317509780000364, + "mean": 9.020872204200169, + "median": 8.954920156999833, + "min": 8.87553689700053, + "rounds": 10, + "stddev": 0.15860320382748683 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.39739922900025704, + "iterations": 1, + "max": 1.596740101999785, + "mean": 1.2538246773002357, + "median": 1.1523321805002524, + "min": 1.0067692990005526, + "rounds": 10, + "stddev": 0.22604122481213654 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.015732269000181986, + "iterations": 1, + "max": 0.43896487100028025, + "mean": 0.4068161139000949, + "median": 0.402179771000192, + "min": 0.38958073999947374, + "rounds": 10, + "stddev": 0.01511178417901699 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.1075060269995447, + "iterations": 1, + "max": 11.733495036998647, + "mean": 11.409412549699663, + "median": 11.406417558999237, + "min": 11.145471429999816, + "rounds": 10, + "stddev": 0.14956303525236064 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.018893256000410474, + "iterations": 1, + "max": 0.4194395229997099, + "mean": 0.40799655209975755, + "median": 0.4117956469995079, + "min": 0.38659930599988, + "rounds": 10, + "stddev": 0.011438685768189766 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.16408438800044678, + "iterations": 1, + "max": 0.8389295440001661, + "mean": 0.5827380278999044, + "median": 0.5586476964999747, + "min": 0.443636360999335, + "rounds": 10, + "stddev": 0.11882012247572926 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.48821563300043636, + "iterations": 1, + "max": 12.375418936000642, + "mean": 11.80373706850005, + "median": 11.673361287000262, + "min": 11.542769956000484, + "rounds": 10, + "stddev": 0.2916123208418184 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.0708471420002752, + "iterations": 1, + "max": 7.805387676000464, + "mean": 7.6958962384002, + "median": 7.688380170500295, + "min": 7.60796725199998, + "rounds": 10, + "stddev": 0.0593984045756101 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 255.59992118500122, + "mean": 255.59992118500122, + "median": 255.59992118500122, + "min": 255.59992118500122, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 404.75491659399995, + "mean": 404.75491659399995, + "median": 404.75491659399995, + "min": 404.75491659399995, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.008887690999472397, + "iterations": 1, + "max": 0.2598420429994803, + "mean": 0.2557109386000775, + "median": 0.25848950799991144, + "min": 0.2477988179998647, + "rounds": 10, + "stddev": 0.004763992633197694 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.008116017999782343, + "iterations": 1, + "max": 0.6469659609992959, + "mean": 0.6280186311996658, + "median": 0.6278817804995924, + "min": 0.6125018559996533, + "rounds": 10, + "stddev": 0.010131677559033747 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.0383760450004047, + "iterations": 1, + "max": 9.65408798899989, + "mean": 9.564837040300336, + "median": 9.60569810600009, + "min": 9.314749659999507, + "rounds": 10, + "stddev": 0.11193064641487754 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.5531405609999638, + "iterations": 1, + "max": 4.463075115999345, + "mean": 3.140352721799809, + "median": 3.0186191819998385, + "min": 2.216118827000173, + "rounds": 10, + "stddev": 0.6100003203183946 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.5071025110000846, + "iterations": 1, + "max": 4.048250541000016, + "mean": 2.06445964049999, + "median": 1.9266086555003312, + "min": 1.4979157669995402, + "rounds": 10, + "stddev": 0.7429974362635847 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.6373983489993407, + "iterations": 1, + "max": 26.528215549999004, + "mean": 25.268741459999728, + "median": 25.16339927699937, + "min": 24.39353708499948, + "rounds": 10, + "stddev": 0.6558888251827933 + } + }, + "started": "2022-01-11T10:41:57.168508+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-chatty.json new file mode 100644 index 00000000..3a04d2f2 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-chatty.json @@ -0,0 +1,636 @@ +{ + "ended": "2022-01-11T09:55:04.538488+00:00", + "id": "run-2022-01-11T08_57_55_678_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 15546.51666666667, + "max": 47401.26666666667, + "mean": 31404.745724961882, + "median": 38476.200000000004, + "min": 1726.3333333333333, + "non_zero_mean": 31404.745724961882, + "non_zero_median": 38476.200000000004, + "percentile25": 23952.216666666667, + "percentile75": 39498.73333333334, + "percentile90": 40485.433333333334, + "percentile99": 45203.48666666665, + "percentile999": 47181.48866666668, + "range": 45674.933333333334, + "samples": 66, + "stdev": 12596.728757995941, + "sum": 2072713.2178474846 + } + }, + "cpu": { + "system": { + "iqr": 0.4050167977603011, + "max": 1.069999999999997, + "mean": 0.8352929202191696, + "median": 1.0069999999999997, + "min": 0.034666666666668296, + "non_zero_mean": 0.8352929202191696, + "non_zero_median": 1.0069999999999997, + "percentile25": 0.6288165355730323, + "percentile75": 1.0338333333333334, + "percentile90": 1.043333333333333, + "percentile99": 1.0604666666666667, + "percentile999": 1.069046666666664, + "range": 1.0353333333333286, + "samples": 66, + "stdev": 0.3053131720291275, + "sum": 55.1293327344652 + }, + "user": { + "iqr": 2.528605919210782, + "max": 6.553999999999997, + "mean": 5.005474589406505, + "median": 6.267666666666662, + "min": 0.1953333333333262, + "non_zero_mean": 5.005474589406505, + "non_zero_median": 6.267666666666662, + "percentile25": 3.856227414122558, + "percentile75": 6.38483333333334, + "percentile90": 6.419666666666673, + "percentile99": 6.508499999999997, + "percentile999": 6.549449999999997, + "range": 6.358666666666671, + "samples": 66, + "stdev": 1.9959442441846622, + "sum": 330.36132290082935 + } + }, + "entropy_available_bits": { + "iqr": 7.316666666666666, + "max": 217.26666666666668, + "mean": 23.108085594365882, + "median": 6.233333333333333, + "min": 0.0, + "non_zero_mean": 30.502672984562967, + "non_zero_median": 7.466666666666667, + "percentile25": 1.0833333333333335, + "percentile75": 8.4, + "percentile90": 15.700286685779052, + "percentile99": 215.35999999999999, + "percentile999": 217.07600000000002, + "range": 217.26666666666668, + "samples": 66, + "stdev": 57.55213556916699, + "sum": 1525.1336492281478 + }, + "memory": { + "used": { + "iqr": 1485027328.0, + "max": 7619444736.0, + "mean": 6402810973.090909, + "median": 6793224192.0, + "min": 3443073024.0, + "non_zero_mean": 6402810973.090909, + "non_zero_median": 6793224192.0, + "percentile25": 5640712192.0, + "percentile75": 7125739520.0, + "percentile90": 7318992896.0, + "percentile99": 7616982016.0, + "percentile999": 7619198464.0, + "range": 4176371712.0, + "samples": 66, + "stdev": 962301017.4341196, + "sum": 422585524224.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.006060740749720015, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.20000444474076048, + "non_zero_median": 0.20000444474076048, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.18000577816298788, + "percentile999": 0.25800057781629915, + "range": 0.26666666666666666, + "samples": 66, + "stdev": 0.036472731504210834, + "sum": 0.40000888948152097 + } + }, + "writes_completed": { + "total": { + "iqr": 2409.5333333333333, + "max": 4942.933333333333, + "mean": 3450.37687625289, + "median": 4545.866666666667, + "min": 7.6000000000000005, + "non_zero_mean": 3450.37687625289, + "non_zero_median": 4545.866666666667, + "percentile25": 2278.3833333333337, + "percentile75": 4687.916666666667, + "percentile90": 4779.266666666666, + "percentile99": 4908.396666666666, + "percentile999": 4939.479666666667, + "range": 4935.333333333333, + "samples": 66, + "stdev": 1636.0188763145823, + "sum": 227724.8738326907 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 244.83333333333348, + "max": 2277.266666666667, + "mean": 1605.851515151515, + "median": 1708.8333333333335, + "min": 535.3333333333334, + "non_zero_mean": 1605.851515151515, + "non_zero_median": 1708.8333333333335, + "percentile25": 1538.8666666666666, + "percentile75": 1783.7, + "percentile90": 1875.3666666666668, + "percentile99": 2169.7133333333327, + "percentile999": 2266.511333333334, + "range": 1741.9333333333334, + "samples": 66, + "stdev": 351.80181343739537, + "sum": 105986.19999999997 + } + }, + "cpu": { + "system": { + "iqr": 0.035666666666666846, + "max": 0.2366666666666669, + "mean": 0.14006060606060608, + "median": 0.15166666666666706, + "min": 0.01799999999999997, + "non_zero_mean": 0.14006060606060608, + "non_zero_median": 0.15166666666666706, + "percentile25": 0.12683333333333321, + "percentile75": 0.16250000000000006, + "percentile90": 0.18433333333333324, + "percentile99": 0.22713333333333344, + "percentile999": 0.2357133333333336, + "range": 0.21866666666666695, + "samples": 66, + "stdev": 0.046455452757551426, + "sum": 9.244000000000002 + }, + "user": { + "iqr": 0.04383333333333264, + "max": 0.5666666666666667, + "mean": 0.21811111111111112, + "median": 0.1739999999999997, + "min": 0.02266666666666642, + "non_zero_mean": 0.21811111111111112, + "non_zero_median": 0.1739999999999997, + "percentile25": 0.1546666666666668, + "percentile75": 0.19849999999999945, + "percentile90": 0.5056666666666677, + "percentile99": 0.5653666666666667, + "percentile999": 0.5665366666666667, + "range": 0.5440000000000003, + "samples": 66, + "stdev": 0.14474435268889654, + "sum": 14.39533333333333 + } + }, + "entropy_available_bits": { + "iqr": 1.4666666666666668, + "max": 185.0, + "mean": 9.555555555555555, + "median": 2.3666666666666667, + "min": 0.4666666666666667, + "non_zero_mean": 9.555555555555555, + "non_zero_median": 2.3666666666666667, + "percentile25": 1.4, + "percentile75": 2.8666666666666667, + "percentile90": 4.033333333333333, + "percentile99": 173.90666666666658, + "percentile999": 183.8906666666667, + "range": 184.53333333333333, + "samples": 66, + "stdev": 34.00017965429325, + "sum": 630.6666666666663 + }, + "memory": { + "used": { + "iqr": 38436864.0, + "max": 474791936.0, + "mean": 437894919.75757575, + "median": 450160640.0, + "min": 361709568.0, + "non_zero_mean": 437894919.75757575, + "non_zero_median": 450160640.0, + "percentile25": 420850688.0, + "percentile75": 459287552.0, + "percentile90": 464885760.0, + "percentile99": 473681715.2, + "percentile999": 474680913.92, + "range": 113082368.0, + "samples": 66, + "stdev": 31276824.72442098, + "sum": 28901064704.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1584196.2666666666, + "max": 3389303.466666667, + "mean": 794144.0646464647, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2015904.164102564, + "non_zero_median": 2096059.7333333334, + "percentile25": 0.0, + "percentile75": 1584196.2666666666, + "percentile90": 3272977.0666666664, + "percentile99": 3381138.7733333334, + "percentile999": 3388486.9973333334, + "range": 3389303.466666667, + "samples": 66, + "stdev": 1215399.683733551, + "sum": 52413508.266666666 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.039253, + "mean": 0.001451060606060606, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.01596166666666667, + "non_zero_median": 0.012705500000000002, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.02392209999999987, + "percentile999": 0.03771991000000006, + "range": 0.039253, + "samples": 66, + "stdev": 0.00567734999091542, + "sum": 0.09577000000000001 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 0.295136, + "mean": 0.009138015151515152, + "median": 0.0, + "min": -0.001372, + "non_zero_mean": 0.046393000000000004, + "non_zero_median": 0.013245, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0107895, + "percentile99": 0.18771699999999905, + "percentile999": 0.28439410000000037, + "range": 0.296508, + "samples": 66, + "stdev": 0.04029732323368101, + "sum": 0.6031089999999999 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 11.066666666666666, + "mean": 0.2737373737373737, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9.033333333333333, + "non_zero_median": 9.033333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 8.42333333333331, + "percentile999": 10.802333333333342, + "range": 11.066666666666666, + "samples": 66, + "stdev": 1.6006036804693853, + "sum": 18.066666666666666 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9985.066666666668, + "mean": 302.2878787878788, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9975.5, + "non_zero_median": 9975.5, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9972.63, + "percentile999": 9983.823, + "range": 9985.066666666668, + "samples": 66, + "stdev": 1723.1029170233085, + "sum": 19951.0 + }, + "pg_stat_database_blks_hit": { + "iqr": 4099.066666666666, + "max": 39130.333333333336, + "mean": 21320.672727272726, + "median": 22456.2, + "min": 4410.0, + "non_zero_mean": 21320.672727272726, + "non_zero_median": 22456.2, + "percentile25": 19724.516666666666, + "percentile75": 23823.583333333332, + "percentile90": 25665.3, + "percentile99": 37096.396666666646, + "percentile999": 38926.93966666667, + "range": 34720.333333333336, + "samples": 66, + "stdev": 6257.727278290975, + "sum": 1407164.3999999997 + }, + "pg_stat_database_blks_read": { + "iqr": 69.08333333333334, + "max": 150.86666666666667, + "mean": 96.9030303030303, + "median": 125.8, + "min": 0.0, + "non_zero_mean": 98.39384615384616, + "non_zero_median": 127.86666666666666, + "percentile25": 66.13333333333333, + "percentile75": 135.21666666666667, + "percentile90": 140.56666666666666, + "percentile99": 149.48, + "percentile999": 150.728, + "range": 150.86666666666667, + "samples": 66, + "stdev": 50.466361387578075, + "sum": 6395.5999999999985 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 69.08333333333334, + "max": 150.86666666666667, + "mean": 96.9030303030303, + "median": 125.8, + "min": 0.0, + "non_zero_mean": 98.39384615384616, + "non_zero_median": 127.86666666666666, + "percentile25": 66.13333333333333, + "percentile75": 135.21666666666667, + "percentile90": 140.56666666666666, + "percentile99": 149.48, + "percentile999": 150.728, + "range": 150.86666666666667, + "samples": 66, + "stdev": 50.466361387578075, + "sum": 6395.5999999999985 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.6, + "mean": 0.20505050505050504, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8458333333333333, + "non_zero_median": 0.9666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.5133333333333325, + "percentile999": 1.5913333333333337, + "range": 1.6, + "samples": 66, + "stdev": 0.4171225167635931, + "sum": 13.533333333333333 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2305.866666666666, + "max": 25079.733333333334, + "mean": 7094.740404040404, + "median": 5358.4, + "min": 3012.6666666666665, + "non_zero_mean": 7094.740404040404, + "non_zero_median": 5358.4, + "percentile25": 5008.3, + "percentile75": 7314.166666666666, + "percentile90": 12739.5, + "percentile99": 24277.633333333328, + "percentile999": 24999.523333333338, + "range": 22067.066666666666, + "samples": 66, + "stdev": 4214.379145817492, + "sum": 468252.8666666668 + }, + "pg_stat_database_tup_inserted": { + "iqr": 373.80000000000007, + "max": 795.5333333333333, + "mean": 515.3969696969697, + "median": 667.5666666666666, + "min": 0.0, + "non_zero_mean": 531.503125, + "non_zero_median": 676.7333333333333, + "percentile25": 347.9, + "percentile75": 721.7, + "percentile90": 745.8666666666667, + "percentile99": 793.9300000000001, + "percentile999": 795.3729999999999, + "range": 795.5333333333333, + "samples": 66, + "stdev": 267.540970181613, + "sum": 34016.2 + }, + "pg_stat_database_tup_returned": { + "iqr": 2237.7, + "max": 31097.6, + "mean": 8938.339393939394, + "median": 6684.9, + "min": 4071.866666666667, + "non_zero_mean": 8938.339393939394, + "non_zero_median": 6684.9, + "percentile25": 6218.083333333333, + "percentile75": 8455.783333333333, + "percentile90": 17159.666666666668, + "percentile99": 30978.346666666665, + "percentile999": 31085.674666666666, + "range": 27025.73333333333, + "samples": 66, + "stdev": 5494.1272290001925, + "sum": 589930.4 + }, + "pg_stat_database_tup_updated": { + "iqr": 6.25, + "max": 136.26666666666668, + "mean": 9.911111111111111, + "median": 0.3666666666666667, + "min": 0.0, + "non_zero_mean": 12.57948717948718, + "non_zero_median": 1.6, + "percentile25": 0.06666666666666667, + "percentile75": 6.316666666666666, + "percentile90": 16.833333333333336, + "percentile99": 127.6866666666666, + "percentile999": 135.4086666666667, + "range": 136.26666666666668, + "samples": 66, + "stdev": 25.63387271169595, + "sum": 654.1333333333334 + }, + "pg_stat_database_xact_commit": { + "iqr": 9.450000000000003, + "max": 140.93333333333334, + "mean": 57.99292929292929, + "median": 50.06666666666667, + "min": 8.733333333333333, + "non_zero_mean": 57.99292929292929, + "non_zero_median": 50.06666666666667, + "percentile25": 47.516666666666666, + "percentile75": 56.96666666666667, + "percentile90": 95.5, + "percentile99": 133.0466666666666, + "percentile999": 140.1446666666667, + "range": 132.20000000000002, + "samples": 66, + "stdev": 26.626832116348137, + "sum": 3827.5333333333333 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 4.400000000000004 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.00404040404040404, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.09333333333333181, + "percentile999": 0.24933333333333393, + "range": 0.26666666666666666, + "samples": 66, + "stdev": 0.03282439759448873, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 27.366666666666674, + "max": 129.26666666666665, + "mean": 76.21010101010101, + "median": 77.63333333333333, + "min": 5.2, + "non_zero_mean": 76.21010101010101, + "non_zero_median": 77.63333333333333, + "percentile25": 64.4, + "percentile75": 91.76666666666668, + "percentile90": 108.73333333333332, + "percentile99": 127.96666666666665, + "percentile999": 129.13666666666666, + "range": 124.06666666666665, + "samples": 66, + "stdev": 27.36546764586382, + "sum": 5029.866666666665 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,8,9,11,10,14,18,20,19,21,25,24,22,23,26,27,28,29,30,31,32,34,35,33,36,37,40,38,39,41,42,43,44,45,46,47,50,48,49,52,51,53,55,54,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-11T09:35:51.318463Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "PASS", + "results": { + "created": "2022-01-11T09:35:52.107911Z 2022-01-11T09:35:52.022080Z 2022-01-11T09:35:51.902936Z 2022-01-11T09:35:51.411860Z 2022-01-11T09:35:51.347572Z 2022-01-11T09:35:51.344514Z 2022-01-11T09:35:51.318463Z 2022-01-11T09:35:53.635187Z 2022-01-11T09:35:52.216564Z 2022-01-11T09:35:52.128818Z 2022-01-11T09:40:21.722202Z 2022-01-11T09:40:21.513147Z 2022-01-11T09:40:21.181067Z 2022-01-11T09:40:21.173518Z 2022-01-11T09:40:21.052183Z 2022-01-11T09:40:21.040798Z 2022-01-11T09:40:20.942083Z 2022-01-11T09:40:20.938007Z 2022-01-11T09:40:20.936661Z 2022-01-11T09:40:20.936598Z 2022-01-11T09:43:27.298452Z 2022-01-11T09:43:27.206109Z 2022-01-11T09:43:26.777328Z 2022-01-11T09:43:26.766822Z 2022-01-11T09:43:26.755264Z 2022-01-11T09:43:26.733643Z 2022-01-11T09:43:26.718409Z 2022-01-11T09:43:26.699226Z 2022-01-11T09:43:26.630809Z 2022-01-11T09:43:26.622281Z 2022-01-11T09:46:31.431460Z 2022-01-11T09:46:31.418405Z 2022-01-11T09:46:30.984299Z 2022-01-11T09:46:30.957853Z 2022-01-11T09:46:30.922692Z 2022-01-11T09:46:30.841809Z 2022-01-11T09:46:30.837244Z 2022-01-11T09:46:30.833342Z 2022-01-11T09:46:30.832604Z 2022-01-11T09:46:30.830980Z 2022-01-11T09:49:35.394301Z 2022-01-11T09:49:35.343382Z 2022-01-11T09:49:34.851532Z 2022-01-11T09:49:34.829813Z 2022-01-11T09:49:34.819016Z 2022-01-11T09:49:34.813102Z 2022-01-11T09:49:34.809733Z 2022-01-11T09:49:34.743206Z 2022-01-11T09:49:34.742961Z 2022-01-11T09:49:34.738814Z", + "created_diff": 1.0068932, + "ended": "2022-01-11T09:52:14.233554Z", + "error_job_ids": " []", + "event_first": "2022-01-11T09:36:02.926650Z 2022-01-11T09:37:46.584725Z 2022-01-11T09:36:02.661423Z 2022-01-11T09:37:10.673576Z 2022-01-11T09:35:58.340750Z 2022-01-11T09:36:49.006825Z 2022-01-11T09:35:57.088330Z 2022-01-11T09:37:28.534798Z 2022-01-11T09:36:07.166715Z 2022-01-11T09:36:58.801646Z 2022-01-11T09:40:42.787630Z 2022-01-11T09:40:43.519963Z 2022-01-11T09:40:42.658594Z 2022-01-11T09:40:42.171126Z 2022-01-11T09:40:27.943766Z 2022-01-11T09:40:27.149654Z 2022-01-11T09:40:26.825349Z 2022-01-11T09:40:26.674993Z 2022-01-11T09:40:26.580915Z 2022-01-11T09:40:26.429802Z 2022-01-11T09:43:44.751617Z 2022-01-11T09:43:47.097839Z 2022-01-11T09:43:36.695039Z 2022-01-11T09:43:35.367198Z 2022-01-11T09:43:34.629824Z 2022-01-11T09:43:34.784828Z 2022-01-11T09:43:33.354829Z 2022-01-11T09:43:32.793926Z 2022-01-11T09:43:32.598484Z 2022-01-11T09:43:33.036148Z 2022-01-11T09:46:48.949500Z 2022-01-11T09:46:51.049014Z 2022-01-11T09:46:38.432544Z 2022-01-11T09:46:39.702930Z 2022-01-11T09:46:42.553646Z 2022-01-11T09:46:39.403795Z 2022-01-11T09:46:37.156318Z 2022-01-11T09:46:37.452060Z 2022-01-11T09:46:37.403067Z 2022-01-11T09:46:37.172747Z 2022-01-11T09:49:52.282256Z 2022-01-11T09:49:53.740683Z 2022-01-11T09:49:44.893551Z 2022-01-11T09:49:43.247727Z 2022-01-11T09:49:42.002048Z 2022-01-11T09:49:42.770844Z 2022-01-11T09:49:43.456544Z 2022-01-11T09:49:44.985565Z 2022-01-11T09:49:41.250900Z 2022-01-11T09:49:41.325792Z", + "event_last": "2022-01-11T09:38:36.096730Z 2022-01-11T09:39:54.423617Z 2022-01-11T09:38:37.049433Z 2022-01-11T09:39:31.297887Z 2022-01-11T09:38:32.159163Z 2022-01-11T09:39:18.045387Z 2022-01-11T09:38:30.278303Z 2022-01-11T09:39:43.551731Z 2022-01-11T09:38:39.391885Z 2022-01-11T09:39:25.075577Z 2022-01-11T09:43:03.167653Z 2022-01-11T09:43:02.493715Z 2022-01-11T09:43:01.833233Z 2022-01-11T09:43:02.821342Z 2022-01-11T09:42:52.669372Z 2022-01-11T09:42:52.845182Z 2022-01-11T09:42:51.867894Z 2022-01-11T09:42:51.997366Z 2022-01-11T09:42:51.673376Z 2022-01-11T09:42:51.997366Z 2022-01-11T09:46:06.255910Z 2022-01-11T09:46:07.586100Z 2022-01-11T09:46:01.269128Z 2022-01-11T09:46:00.895815Z 2022-01-11T09:45:59.426457Z 2022-01-11T09:46:01.578622Z 2022-01-11T09:45:59.592095Z 2022-01-11T09:45:59.775414Z 2022-01-11T09:45:58.923520Z 2022-01-11T09:45:59.456310Z 2022-01-11T09:49:10.283663Z 2022-01-11T09:49:12.357633Z 2022-01-11T09:49:01.340926Z 2022-01-11T09:49:05.156175Z 2022-01-11T09:49:05.101700Z 2022-01-11T09:49:04.295295Z 2022-01-11T09:49:02.105906Z 2022-01-11T09:49:03.436442Z 2022-01-11T09:49:01.048050Z 2022-01-11T09:49:02.435396Z 2022-01-11T09:52:11.399177Z 2022-01-11T09:52:14.233554Z 2022-01-11T09:52:06.550781Z 2022-01-11T09:52:08.739895Z 2022-01-11T09:52:02.743988Z 2022-01-11T09:52:09.047639Z 2022-01-11T09:52:04.473455Z 2022-01-11T09:52:08.782126Z 2022-01-11T09:52:04.083509Z 2022-01-11T09:52:07.911120Z", + "failed_job_ids": " []", + "finished": "2022-01-11T09:38:40.331351Z 2022-01-11T09:39:58.358299Z 2022-01-11T09:38:40.853482Z 2022-01-11T09:39:34.659200Z 2022-01-11T09:38:36.196642Z 2022-01-11T09:39:21.475534Z 2022-01-11T09:38:33.985032Z 2022-01-11T09:39:47.795517Z 2022-01-11T09:38:43.543899Z 2022-01-11T09:39:28.525220Z 2022-01-11T09:43:05.865772Z 2022-01-11T09:43:06.231155Z 2022-01-11T09:43:05.464722Z 2022-01-11T09:43:07.110634Z 2022-01-11T09:42:56.267348Z 2022-01-11T09:42:56.082421Z 2022-01-11T09:42:55.852239Z 2022-01-11T09:42:55.697017Z 2022-01-11T09:42:55.798907Z 2022-01-11T09:42:55.619709Z 2022-01-11T09:46:10.148205Z 2022-01-11T09:46:11.825382Z 2022-01-11T09:46:05.399739Z 2022-01-11T09:46:04.392846Z 2022-01-11T09:46:03.090438Z 2022-01-11T09:46:05.275540Z 2022-01-11T09:46:03.634817Z 2022-01-11T09:46:03.673163Z 2022-01-11T09:46:03.375405Z 2022-01-11T09:46:03.414357Z 2022-01-11T09:49:11.863393Z 2022-01-11T09:49:13.985366Z 2022-01-11T09:49:05.644193Z 2022-01-11T09:49:09.401534Z 2022-01-11T09:49:08.394553Z 2022-01-11T09:49:07.919693Z 2022-01-11T09:49:05.651995Z 2022-01-11T09:49:06.805852Z 2022-01-11T09:49:04.829207Z 2022-01-11T09:49:06.791042Z 2022-01-11T09:52:15.497466Z 2022-01-11T09:52:17.718297Z 2022-01-11T09:52:10.428946Z 2022-01-11T09:52:12.483583Z 2022-01-11T09:52:07.060458Z 2022-01-11T09:52:12.411617Z 2022-01-11T09:52:07.836554Z 2022-01-11T09:52:12.128928Z 2022-01-11T09:52:07.736326Z 2022-01-11T09:52:11.637579Z", + "finished_diff": 24.8826268, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 244.407424, + "mean": 162.83470018000006, + "min": 150.804008 + }, + "jobs_events_duration": { + "max": 154.38801, + "mean": 144.2430704, + "min": 127.838892 + }, + "jobs_events_lag": { + "max": -1.57973, + "mean": -3.7029712200000007, + "min": -4.451885 + }, + "jobs_waiting": { + "max": 13.076782, + "mean": 2.7275701, + "min": 0.841683 + }, + "modified": "2022-01-11T09:35:53.526219Z 2022-01-11T09:35:53.490321Z 2022-01-11T09:35:53.434664Z 2022-01-11T09:35:51.961458Z 2022-01-11T09:35:51.932709Z 2022-01-11T09:35:51.903132Z 2022-01-11T09:35:51.866275Z 2022-01-11T09:35:54.581642Z 2022-01-11T09:35:53.593529Z 2022-01-11T09:35:53.558106Z 2022-01-11T09:40:33.695605Z 2022-01-11T09:40:33.623397Z 2022-01-11T09:40:33.536748Z 2022-01-11T09:40:33.439790Z 2022-01-11T09:40:21.624100Z 2022-01-11T09:40:21.589220Z 2022-01-11T09:40:21.557265Z 2022-01-11T09:40:21.523366Z 2022-01-11T09:40:21.487348Z 2022-01-11T09:40:21.450473Z 2022-01-11T09:43:35.099171Z 2022-01-11T09:43:34.949543Z 2022-01-11T09:43:27.417339Z 2022-01-11T09:43:27.384006Z 2022-01-11T09:43:27.349473Z 2022-01-11T09:43:27.312921Z 2022-01-11T09:43:27.276616Z 2022-01-11T09:43:27.244164Z 2022-01-11T09:43:27.211921Z 2022-01-11T09:43:27.179682Z 2022-01-11T09:46:36.340197Z 2022-01-11T09:46:36.298762Z 2022-01-11T09:46:31.627610Z 2022-01-11T09:46:31.596977Z 2022-01-11T09:46:31.566597Z 2022-01-11T09:46:31.525014Z 2022-01-11T09:46:31.486364Z 2022-01-11T09:46:31.455536Z 2022-01-11T09:46:31.424783Z 2022-01-11T09:46:31.392449Z 2022-01-11T09:49:37.803416Z 2022-01-11T09:49:37.749756Z 2022-01-11T09:49:35.542496Z 2022-01-11T09:49:35.505540Z 2022-01-11T09:49:35.468068Z 2022-01-11T09:49:35.433014Z 2022-01-11T09:49:35.399205Z 2022-01-11T09:49:35.368341Z 2022-01-11T09:49:35.336240Z 2022-01-11T09:49:35.303976Z", + "modified_diff": 6.0654352000000005, + "started": "2022-01-11T09:35:54.040706Z 2022-01-11T09:35:53.950875Z 2022-01-11T09:35:53.883772Z 2022-01-11T09:35:52.482830Z 2022-01-11T09:35:52.335413Z 2022-01-11T09:35:52.395912Z 2022-01-11T09:35:52.160146Z 2022-01-11T09:35:54.846150Z 2022-01-11T09:35:54.275678Z 2022-01-11T09:35:54.150882Z 2022-01-11T09:40:34.494120Z 2022-01-11T09:40:34.570622Z 2022-01-11T09:40:34.257849Z 2022-01-11T09:40:34.155683Z 2022-01-11T09:40:22.231235Z 2022-01-11T09:40:22.151480Z 2022-01-11T09:40:22.053334Z 2022-01-11T09:40:21.980316Z 2022-01-11T09:40:21.896833Z 2022-01-11T09:40:21.792508Z 2022-01-11T09:43:35.804501Z 2022-01-11T09:43:35.530287Z 2022-01-11T09:43:28.285398Z 2022-01-11T09:43:28.220551Z 2022-01-11T09:43:28.054315Z 2022-01-11T09:43:27.945683Z 2022-01-11T09:43:27.847437Z 2022-01-11T09:43:27.770904Z 2022-01-11T09:43:27.671022Z 2022-01-11T09:43:27.603394Z 2022-01-11T09:46:36.796932Z 2022-01-11T09:46:36.662191Z 2022-01-11T09:46:32.441040Z 2022-01-11T09:46:32.304395Z 2022-01-11T09:46:32.217093Z 2022-01-11T09:46:32.113731Z 2022-01-11T09:46:32.036778Z 2022-01-11T09:46:31.958276Z 2022-01-11T09:46:31.878916Z 2022-01-11T09:46:31.817290Z 2022-01-11T09:49:38.251045Z 2022-01-11T09:49:38.165035Z 2022-01-11T09:49:36.424029Z 2022-01-11T09:49:36.344322Z 2022-01-11T09:49:36.256450Z 2022-01-11T09:49:36.155446Z 2022-01-11T09:49:36.067735Z 2022-01-11T09:49:35.982121Z 2022-01-11T09:49:35.906098Z 2022-01-11T09:49:35.816806Z", + "started_diff": 6.215821199999999, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-11T09:55:01.835200+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-launching.json new file mode 100644 index 00000000..17af73c8 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T08_57_55_678_0000-launching.json @@ -0,0 +1,634 @@ +{ + "ended": "2022-01-11T10:41:07.133929+00:00", + "id": "run-2022-01-11T08_57_55_678_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17946.499999999996, + "max": 26964.066666666666, + "mean": 9704.640506329113, + "median": 3806.133333333333, + "min": 1464.0666666666666, + "non_zero_mean": 9704.640506329113, + "non_zero_median": 3806.133333333333, + "percentile25": 1746.6666666666665, + "percentile75": 19693.166666666664, + "percentile90": 22849.613333333335, + "percentile99": 26890.590666666667, + "percentile999": 26956.71906666667, + "range": 25500.0, + "samples": 79, + "stdev": 9184.82238847001, + "sum": 766666.6000000001 + } + }, + "cpu": { + "system": { + "iqr": 1.4179999999999988, + "max": 1.8380000000000014, + "mean": 0.6847932489451476, + "median": 0.2326666666666654, + "min": 0.03133333333333137, + "non_zero_mean": 0.6847932489451476, + "non_zero_median": 0.2326666666666654, + "percentile25": 0.038333333333333525, + "percentile75": 1.4563333333333324, + "percentile90": 1.6600000000000033, + "percentile99": 1.7995200000000022, + "percentile999": 1.834152000000002, + "range": 1.80666666666667, + "samples": 79, + "stdev": 0.7132941763831393, + "sum": 54.098666666666674 + }, + "user": { + "iqr": 5.80166666666665, + "max": 6.513999999999972, + "mean": 2.848438818565401, + "median": 0.9613333333333381, + "min": 0.07399999999996303, + "non_zero_mean": 2.848438818565401, + "non_zero_median": 0.9613333333333381, + "percentile25": 0.30366666666668224, + "percentile75": 6.105333333333332, + "percentile90": 6.337199999999999, + "percentile99": 6.497360000000005, + "percentile999": 6.5123359999999755, + "range": 6.440000000000008, + "samples": 79, + "stdev": 2.781519697532546, + "sum": 225.0266666666667 + } + }, + "entropy_available_bits": { + "iqr": 4.833333333333334, + "max": 212.93333333333334, + "mean": 23.360337552742617, + "median": 1.9333333333333333, + "min": 0.0, + "non_zero_mean": 32.3766081871345, + "non_zero_median": 4.4, + "percentile25": 0.0, + "percentile75": 4.833333333333334, + "percentile90": 47.37333333333391, + "percentile99": 211.32133333333334, + "percentile999": 212.77213333333336, + "range": 212.93333333333334, + "samples": 79, + "stdev": 63.44662518351977, + "sum": 1845.466666666667 + }, + "memory": { + "used": { + "iqr": 5283084288.0, + "max": 18736689152.0, + "mean": 8740967333.265823, + "median": 6463746048.0, + "min": 5272551424.0, + "non_zero_mean": 8740967333.265823, + "non_zero_median": 6463746048.0, + "percentile25": 6034358272.0, + "percentile75": 11317442560.0, + "percentile90": 15098136985.6, + "percentile99": 18052496015.36, + "percentile999": 18668269838.33601, + "range": 13464137728.0, + "samples": 79, + "stdev": 3828813234.460457, + "sum": 690536419328.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 4.0, + "mean": 0.05738396624472574, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.511111111111111, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.0879999999999959, + "percentile999": 3.708800000000042, + "range": 4.0, + "samples": 79, + "stdev": 0.4512387732142191, + "sum": 4.533333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 759.5000000000001, + "max": 1644.4, + "mean": 447.0396624472574, + "median": 98.46666666666665, + "min": 0.0, + "non_zero_mean": 452.7709401709402, + "non_zero_median": 102.1, + "percentile25": 3.6333333333333337, + "percentile75": 763.1333333333334, + "percentile90": 1402.3066666666666, + "percentile99": 1638.94, + "percentile999": 1643.8540000000003, + "range": 1644.4, + "samples": 79, + "stdev": 571.4579121495206, + "sum": 35316.13333333334 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2286.7333333333336, + "max": 3782.4666666666667, + "mean": 1697.620253164557, + "median": 899.4, + "min": 396.8666666666667, + "non_zero_mean": 1697.620253164557, + "non_zero_median": 899.4, + "percentile25": 599.9333333333334, + "percentile75": 2886.666666666667, + "percentile90": 3231.3733333333334, + "percentile99": 3620.434666666666, + "percentile999": 3766.263466666669, + "range": 3385.6, + "samples": 79, + "stdev": 1167.0980822542913, + "sum": 134112.00000000003 + } + }, + "cpu": { + "system": { + "iqr": 0.23300000000000076, + "max": 0.3939999999999988, + "mean": 0.13285232067510547, + "median": 0.051333333333333536, + "min": 0.010000000000001327, + "non_zero_mean": 0.13285232067510547, + "non_zero_median": 0.051333333333333536, + "percentile25": 0.022999999999999923, + "percentile75": 0.25600000000000067, + "percentile90": 0.3071999999999999, + "percentile99": 0.39296000000000053, + "percentile999": 0.39389599999999897, + "range": 0.38399999999999745, + "samples": 79, + "stdev": 0.12331886894663611, + "sum": 10.495333333333333 + }, + "user": { + "iqr": 0.29766666666666686, + "max": 0.7866666666666655, + "mean": 0.20059915611814347, + "median": 0.0726666666666669, + "min": 0.010666666666668333, + "non_zero_mean": 0.20059915611814347, + "non_zero_median": 0.0726666666666669, + "percentile25": 0.03633333333333345, + "percentile75": 0.3340000000000003, + "percentile90": 0.4435999999999987, + "percentile99": 0.680066666666667, + "percentile999": 0.7760066666666672, + "range": 0.7759999999999971, + "samples": 79, + "stdev": 0.19542813238644435, + "sum": 15.847333333333331 + } + }, + "entropy_available_bits": { + "iqr": 3.533333333333333, + "max": 210.53333333333333, + "mean": 11.914767932489452, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 18.101282051282052, + "non_zero_median": 2.3666666666666667, + "percentile25": 0.0, + "percentile75": 3.533333333333333, + "percentile90": 4.533333333333333, + "percentile99": 210.48133333333334, + "percentile999": 210.52813333333333, + "range": 210.53333333333333, + "samples": 79, + "stdev": 46.106689031138586, + "sum": 941.2666666666665 + }, + "memory": { + "used": { + "iqr": 367484928.0, + "max": 1116405760.0, + "mean": 565877228.556962, + "median": 404774912.0, + "min": 360722432.0, + "non_zero_mean": 565877228.556962, + "non_zero_median": 404774912.0, + "percentile25": 371871744.0, + "percentile75": 739356672.0, + "percentile90": 963885465.6, + "percentile99": 1109527183.36, + "percentile999": 1115717902.3360002, + "range": 755683328.0, + "samples": 79, + "stdev": 250927696.43292144, + "sum": 44704301056.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6007.466666666666, + "max": 55860214.333333336, + "mean": 724816.4582278482, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2385854.1750000003, + "non_zero_median": 56524.8, + "percentile25": 0.0, + "percentile75": 6007.466666666666, + "percentile90": 81701.54666666668, + "percentile99": 12430673.841333272, + "percentile999": 51517260.28413396, + "range": 55860214.333333336, + "samples": 79, + "stdev": 6282876.506153419, + "sum": 57260500.199999996 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 1.428583, + "mean": 0.04392806329113924, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2478797857142857, + "non_zero_median": 0.0357335, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.019614200000000005, + "percentile99": 1.1187077199999995, + "percentile999": 1.3975954720000043, + "range": 1.428583, + "samples": 79, + "stdev": 0.20413445855840176, + "sum": 3.470317 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.164759, + "max": 9.665144, + "mean": 0.6090258607594937, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.3746583714285714, + "non_zero_median": 0.371422, + "percentile25": 0.0, + "percentile75": 0.164759, + "percentile90": 1.4287064, + "percentile99": 8.728236859999999, + "percentile999": 9.571453286000013, + "range": 9.665144, + "samples": 79, + "stdev": 1.69866452797808, + "sum": 48.113043000000005 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.7333333333333334, + "mean": 0.14261603375527426, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.8166666666666664, + "non_zero_median": 2.7666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.1613333333333324, + "percentile999": 3.6761333333333415, + "range": 3.7333333333333334, + "samples": 79, + "stdev": 0.6379931299557048, + "sum": 11.266666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9995.266666666666, + "mean": 415.0303797468354, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8196.85, + "non_zero_median": 8268.8, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9990.066666666668, + "percentile999": 9994.746666666666, + "range": 9995.266666666666, + "samples": 79, + "stdev": 1853.8881761793211, + "sum": 32787.4 + }, + "pg_stat_database_blks_hit": { + "iqr": 25225.866666666665, + "max": 101641.53333333334, + "mean": 21592.738396624474, + "median": 10440.6, + "min": 1195.2666666666667, + "non_zero_mean": 21592.738396624474, + "non_zero_median": 10440.6, + "percentile25": 6571.8, + "percentile75": 31797.666666666664, + "percentile90": 51822.44, + "percentile99": 80953.3853333333, + "percentile999": 99572.71853333364, + "range": 100446.26666666668, + "samples": 79, + "stdev": 21236.28442911509, + "sum": 1705826.3333333344 + }, + "pg_stat_database_blks_read": { + "iqr": 3.3, + "max": 13.733333333333333, + "mean": 2.1502109704641352, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 3.466666666666667, + "non_zero_median": 2.533333333333333, + "percentile25": 0.0, + "percentile75": 3.3, + "percentile90": 6.520000000000002, + "percentile99": 12.38133333333333, + "percentile999": 13.598133333333353, + "range": 13.733333333333333, + "samples": 79, + "stdev": 3.2224544865496245, + "sum": 169.86666666666662 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 79, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.3, + "max": 13.733333333333333, + "mean": 2.1502109704641352, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 3.466666666666667, + "non_zero_median": 2.533333333333333, + "percentile25": 0.0, + "percentile75": 3.3, + "percentile90": 6.520000000000002, + "percentile99": 12.38133333333333, + "percentile999": 13.598133333333353, + "range": 13.733333333333333, + "samples": 79, + "stdev": 3.2224544865496245, + "sum": 169.86666666666662 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 2.1333333333333333, + "mean": 0.23122362869198312, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.2177777777777778, + "non_zero_median": 1.2666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.1600000000000004, + "percentile99": 2.029333333333333, + "percentile999": 2.122933333333335, + "range": 2.1333333333333333, + "samples": 79, + "stdev": 0.55293420297974, + "sum": 18.26666666666667 + }, + "pg_stat_database_tup_fetched": { + "iqr": 14648.8, + "max": 51579.6, + "mean": 11546.014345991562, + "median": 5463.8, + "min": 578.8, + "non_zero_mean": 11546.014345991562, + "non_zero_median": 5463.8, + "percentile25": 3392.7666666666664, + "percentile75": 18041.566666666666, + "percentile90": 27945.400000000005, + "percentile99": 41500.283999999985, + "percentile999": 50571.66840000014, + "range": 51000.799999999996, + "samples": 79, + "stdev": 11120.140545596025, + "sum": 912135.1333333333 + }, + "pg_stat_database_tup_inserted": { + "iqr": 27.0, + "max": 105.46666666666667, + "mean": 16.88270042194093, + "median": 4.133333333333334, + "min": 0.0, + "non_zero_mean": 26.674666666666667, + "non_zero_median": 22.233333333333334, + "percentile25": 0.0, + "percentile75": 27.0, + "percentile90": 49.13333333333335, + "percentile99": 91.16666666666666, + "percentile999": 104.03666666666687, + "range": 105.46666666666667, + "samples": 79, + "stdev": 24.25634986665444, + "sum": 1333.7333333333333 + }, + "pg_stat_database_tup_returned": { + "iqr": 21568.166666666668, + "max": 79902.66666666667, + "mean": 18266.68523206751, + "median": 9353.8, + "min": 1374.1333333333334, + "non_zero_mean": 18266.68523206751, + "non_zero_median": 9353.8, + "percentile25": 5904.333333333333, + "percentile75": 27472.5, + "percentile90": 40493.57333333334, + "percentile99": 66420.10666666664, + "percentile999": 78554.41066666687, + "range": 78528.53333333334, + "samples": 79, + "stdev": 17054.279949071533, + "sum": 1443068.1333333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 31.0, + "max": 73.53333333333333, + "mean": 16.96708860759494, + "median": 7.333333333333333, + "min": 0.0, + "non_zero_mean": 18.616666666666667, + "non_zero_median": 13.133333333333333, + "percentile25": 0.26666666666666666, + "percentile75": 31.266666666666666, + "percentile90": 45.733333333333334, + "percentile99": 72.59733333333332, + "percentile999": 73.43973333333335, + "range": 73.53333333333333, + "samples": 79, + "stdev": 19.64804528650031, + "sum": 1340.4000000000005 + }, + "pg_stat_database_xact_commit": { + "iqr": 262.66666666666663, + "max": 622.5333333333333, + "mean": 151.73080168776372, + "median": 57.8, + "min": 2.8, + "non_zero_mean": 151.73080168776372, + "non_zero_median": 57.8, + "percentile25": 8.733333333333334, + "percentile75": 271.4, + "percentile90": 374.61333333333334, + "percentile99": 556.5973333333333, + "percentile999": 615.9397333333343, + "range": 619.7333333333333, + "samples": 79, + "stdev": 163.16146738109782, + "sum": 11986.733333333324 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 79, + "stdev": 0.0, + "sum": 5.2666666666666675 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 2.1333333333333333, + "mean": 0.04219409282700422, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.6666666666666665, + "non_zero_median": 1.6666666666666665, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.4053333333333322, + "percentile999": 2.060533333333344, + "range": 2.1333333333333333, + "samples": 79, + "stdev": 0.2738721425209005, + "sum": 3.333333333333333 + } + }, + "writes_completed": { + "total": { + "iqr": 105.16666666666666, + "max": 174.33333333333331, + "mean": 57.413502109704645, + "median": 30.53333333333333, + "min": 0.26666666666666666, + "non_zero_mean": 57.413502109704645, + "non_zero_median": 30.53333333333333, + "percentile25": 4.9, + "percentile75": 110.06666666666666, + "percentile90": 136.54666666666665, + "percentile99": 167.5213333333333, + "percentile999": 173.6521333333334, + "range": 174.06666666666663, + "samples": 79, + "stdev": 55.43428067067727, + "sum": 4535.666666666665 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,60,62,63,69,66,68,65,70,71,72,75,74,78,81,76,77,79,80,87,90,82,89,84,88,83,97,93,92,91,96,100,101,98,103,99,104,102,105,94,106,107,108,110,114,111,112,109,95,117,116,118,113,121,123,125,119,127,126,115,128,122,133,135,134,137,136,131,138,147,146,148,140,145,142,141,132,149,129,151,124,152,153,155,154,158,157,143,156,159,162,161,163,164,165,160,166,167,168,169,174,170,180,176,173,172,179,175,171,177,178,181,182,183,185,187,184,186,190,192,196,189,188,191,194,193,195,198,203,197,199,205,202,201,207,200,206,204,209,208,211,210,212,216,217,213,215,218,220,222,214,221,228,224,227,229,226,225,230,231,232,219,223,235,238,233,236,241,237,239,242,243,240,249,245,247,251,250,244,234,253,246,255,254,248,257,258,252,261,259,263,260,256,264,265,262,266,267,268,269,270,273,276,275,271,272,278,274,279,283,277,289,294,287,280,290,296,281,288,286,285,282,297,291,284,293,292,298,299,295,300,302,303,305,307,301,310,311,306,313,304,314,316,319,308,309,320,312,317,324,318,315,321,328,323,322,326,325,330,331,332,327,333,334,329,344,343,335,340,337,341,339,336,345,342,348,338,351,349,346,353,354,356,350,359,357,358,355,361,362,347,363,360,352,365,364,366,368,369,367,370,371,373,372,374,376,375,377,379,378,381,380,383,384,382,388,387,392,390,389,386,391,385,394,397,393,400,395,396,401,398,399,405,406,403,402,411,404,412,407,409,413,416,408,415,410,419,417,414,421,424,420,422,423,418,425,427,429,434,432,435,426,430,431,436,428,433,440,441,439,444,438,437,443,442,452,451,445,447,455,446,450,457,461,454,453,460,462,459,464,456,463,449,448,465,458,468,467,466,469,470,473,479,471,476,477,484,475,480,472,478,474,485,481,483,492,488,491,482,486,487,489,490,496,501,502,499,495,498,506,508,505,494,497,500,510,493,509,513,512,507,511,520,503,504,518,515,516,514,517,521,519,530,522,527,523,524,525,532,526,529,528,531,534,542,537,535,536,541,539,538,533,547,540,546,544,548,545,551,553,550,555,554,552,543,560,561,557,556,564,559,563,566,567,562,549,565,558,569,568", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-11T09:56:46.399086Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2022-01-11T09:56:52.170873Z 2022-01-11T09:56:52.005764Z 2022-01-11T09:56:51.968418Z 2022-01-11T09:56:51.879557Z 2022-01-11T09:56:51.825949Z 2022-01-11T09:56:51.725885Z 2022-01-11T09:56:51.669913Z 2022-01-11T09:56:51.647253Z 2022-01-11T09:56:51.532247Z 2022-01-11T09:56:51.216787Z 2022-01-11T09:56:51.114270Z 2022-01-11T09:56:50.920986Z 2022-01-11T09:56:50.886555Z 2022-01-11T09:56:50.774969Z 2022-01-11T09:56:50.689295Z 2022-01-11T09:56:50.097268Z 2022-01-11T09:56:50.019790Z 2022-01-11T09:56:49.990628Z 2022-01-11T09:56:49.744699Z 2022-01-11T09:56:49.707647Z 2022-01-11T09:56:49.647939Z 2022-01-11T09:56:49.610823Z 2022-01-11T09:56:49.257899Z 2022-01-11T09:56:49.180212Z 2022-01-11T09:56:48.898149Z 2022-01-11T09:56:48.890477Z 2022-01-11T09:56:48.734201Z 2022-01-11T09:56:48.708202Z 2022-01-11T09:56:48.656366Z 2022-01-11T09:56:48.026308Z 2022-01-11T09:56:47.500006Z 2022-01-11T09:56:47.192304Z 2022-01-11T09:56:46.921217Z 2022-01-11T09:56:46.581707Z 2022-01-11T09:56:46.399086Z 2022-01-11T09:57:14.871818Z 2022-01-11T09:57:12.885131Z 2022-01-11T09:57:11.021855Z 2022-01-11T09:57:09.532701Z 2022-01-11T09:57:08.363230Z 2022-01-11T09:57:07.313591Z 2022-01-11T09:57:07.171391Z 2022-01-11T09:57:05.535667Z 2022-01-11T09:57:05.128479Z 2022-01-11T09:57:04.106038Z 2022-01-11T09:57:03.840565Z 2022-01-11T09:57:02.886002Z 2022-01-11T09:57:02.252266Z 2022-01-11T09:57:02.206941Z 2022-01-11T09:57:01.470439Z 2022-01-11T09:57:01.232581Z 2022-01-11T09:57:00.381390Z 2022-01-11T09:56:59.911511Z 2022-01-11T09:56:59.152600Z 2022-01-11T09:56:58.417308Z 2022-01-11T09:56:58.161892Z 2022-01-11T09:56:57.934273Z 2022-01-11T09:56:57.931342Z 2022-01-11T09:56:57.871851Z 2022-01-11T09:56:57.713608Z 2022-01-11T09:56:57.650612Z 2022-01-11T09:56:57.630305Z 2022-01-11T09:56:57.435249Z 2022-01-11T09:56:57.168601Z 2022-01-11T09:56:56.982594Z 2022-01-11T09:56:56.964168Z 2022-01-11T09:56:56.871880Z 2022-01-11T09:56:56.771563Z 2022-01-11T09:56:56.769709Z 2022-01-11T09:56:56.750468Z 2022-01-11T09:56:56.536137Z 2022-01-11T09:56:56.135757Z 2022-01-11T09:56:55.855033Z 2022-01-11T09:56:55.787593Z 2022-01-11T09:56:55.768053Z 2022-01-11T09:56:55.703098Z 2022-01-11T09:56:55.586988Z 2022-01-11T09:56:55.566079Z 2022-01-11T09:56:55.536743Z 2022-01-11T09:56:55.528200Z 2022-01-11T09:56:54.980906Z 2022-01-11T09:56:54.792619Z 2022-01-11T09:56:54.765029Z 2022-01-11T09:56:54.722061Z 2022-01-11T09:56:54.665298Z 2022-01-11T09:56:54.586220Z 2022-01-11T09:56:54.466873Z 2022-01-11T09:56:54.257950Z 2022-01-11T09:56:53.813317Z 2022-01-11T09:56:53.798358Z 2022-01-11T09:56:53.778761Z 2022-01-11T09:56:53.758371Z 2022-01-11T09:56:53.721836Z 2022-01-11T09:56:53.492288Z 2022-01-11T09:56:53.346615Z 2022-01-11T09:56:52.795923Z 2022-01-11T09:56:52.720841Z 2022-01-11T09:56:52.573337Z 2022-01-11T09:56:52.549533Z 2022-01-11T09:56:52.442899Z 2022-01-11T10:01:39.371858Z 2022-01-11T10:01:39.258575Z 2022-01-11T10:01:38.427076Z 2022-01-11T10:01:38.041126Z 2022-01-11T10:01:37.619855Z 2022-01-11T10:01:36.791786Z 2022-01-11T10:01:36.689627Z 2022-01-11T10:01:36.437813Z 2022-01-11T10:01:36.406488Z 2022-01-11T10:01:36.149312Z 2022-01-11T10:01:35.855618Z 2022-01-11T10:01:35.741186Z 2022-01-11T10:01:35.317581Z 2022-01-11T10:01:35.254434Z 2022-01-11T10:01:34.223161Z 2022-01-11T10:01:33.892300Z 2022-01-11T10:01:33.679174Z 2022-01-11T10:01:33.581769Z 2022-01-11T10:01:33.470312Z 2022-01-11T10:01:33.167566Z 2022-01-11T10:01:32.944759Z 2022-01-11T10:01:32.917264Z 2022-01-11T10:01:32.755790Z 2022-01-11T10:01:32.753322Z 2022-01-11T10:01:32.706198Z 2022-01-11T10:01:32.645767Z 2022-01-11T10:01:31.525145Z 2022-01-11T10:01:30.904013Z 2022-01-11T10:01:30.869789Z 2022-01-11T10:01:30.820684Z 2022-01-11T10:01:30.681400Z 2022-01-11T10:01:30.652929Z 2022-01-11T10:01:30.581720Z 2022-01-11T10:01:30.360526Z 2022-01-11T10:01:29.589394Z 2022-01-11T10:01:29.383150Z 2022-01-11T10:01:29.373058Z 2022-01-11T10:01:28.822034Z 2022-01-11T10:01:28.772260Z 2022-01-11T10:01:28.004610Z 2022-01-11T10:01:27.967150Z 2022-01-11T10:01:27.899869Z 2022-01-11T10:01:27.869267Z 2022-01-11T10:01:27.865376Z 2022-01-11T10:01:27.546035Z 2022-01-11T10:01:27.478289Z 2022-01-11T10:01:27.439853Z 2022-01-11T10:01:27.288687Z 2022-01-11T10:01:27.140309Z 2022-01-11T10:01:27.069849Z 2022-01-11T10:01:27.067092Z 2022-01-11T10:01:26.659512Z 2022-01-11T10:01:26.260265Z 2022-01-11T10:01:26.213351Z 2022-01-11T10:01:26.054008Z 2022-01-11T10:01:25.892102Z 2022-01-11T10:01:25.808745Z 2022-01-11T10:01:25.763543Z 2022-01-11T10:01:25.543193Z 2022-01-11T10:01:25.085373Z 2022-01-11T10:01:25.045740Z 2022-01-11T10:01:24.899906Z 2022-01-11T10:01:24.734012Z 2022-01-11T10:01:24.708751Z 2022-01-11T10:01:24.577217Z 2022-01-11T10:01:24.513683Z 2022-01-11T10:01:24.428458Z 2022-01-11T10:01:24.405127Z 2022-01-11T10:01:24.347736Z 2022-01-11T10:01:24.291269Z 2022-01-11T10:01:23.939417Z 2022-01-11T10:01:23.858478Z 2022-01-11T10:01:23.649217Z 2022-01-11T10:01:23.499206Z 2022-01-11T10:01:23.393286Z 2022-01-11T10:01:23.326323Z 2022-01-11T10:01:22.994053Z 2022-01-11T10:01:22.929642Z 2022-01-11T10:01:22.900462Z 2022-01-11T10:01:22.732581Z 2022-01-11T10:01:22.732407Z 2022-01-11T10:01:22.719241Z 2022-01-11T10:01:22.700727Z 2022-01-11T10:01:22.366898Z 2022-01-11T10:01:22.303627Z 2022-01-11T10:01:22.243331Z 2022-01-11T10:01:22.226139Z 2022-01-11T10:01:22.100492Z 2022-01-11T10:01:21.965021Z 2022-01-11T10:01:21.722541Z 2022-01-11T10:01:21.687498Z 2022-01-11T10:01:21.614881Z 2022-01-11T10:01:21.574058Z 2022-01-11T10:01:21.336605Z 2022-01-11T10:01:21.145746Z 2022-01-11T10:01:21.099365Z 2022-01-11T10:01:20.992324Z 2022-01-11T10:01:20.927685Z 2022-01-11T10:01:20.926089Z 2022-01-11T10:01:20.922742Z 2022-01-11T10:05:38.401495Z 2022-01-11T10:05:36.939000Z 2022-01-11T10:05:36.844291Z 2022-01-11T10:05:36.838967Z 2022-01-11T10:05:36.459625Z 2022-01-11T10:05:36.419928Z 2022-01-11T10:05:35.581507Z 2022-01-11T10:05:35.537081Z 2022-01-11T10:05:35.316983Z 2022-01-11T10:05:35.074773Z 2022-01-11T10:05:34.875658Z 2022-01-11T10:05:34.605944Z 2022-01-11T10:05:34.450936Z 2022-01-11T10:05:34.307184Z 2022-01-11T10:05:34.244806Z 2022-01-11T10:05:34.049545Z 2022-01-11T10:05:33.695470Z 2022-01-11T10:05:33.572600Z 2022-01-11T10:05:33.552436Z 2022-01-11T10:05:33.542541Z 2022-01-11T10:05:33.473084Z 2022-01-11T10:05:33.410582Z 2022-01-11T10:05:33.372497Z 2022-01-11T10:05:32.805335Z 2022-01-11T10:05:32.635043Z 2022-01-11T10:05:32.285626Z 2022-01-11T10:05:32.237010Z 2022-01-11T10:05:32.174597Z 2022-01-11T10:05:32.162055Z 2022-01-11T10:05:32.007494Z 2022-01-11T10:05:31.922528Z 2022-01-11T10:05:31.793388Z 2022-01-11T10:05:31.738089Z 2022-01-11T10:05:31.646508Z 2022-01-11T10:05:31.538832Z 2022-01-11T10:05:31.491244Z 2022-01-11T10:05:31.115090Z 2022-01-11T10:05:31.087464Z 2022-01-11T10:05:30.899220Z 2022-01-11T10:05:30.840116Z 2022-01-11T10:05:30.631395Z 2022-01-11T10:05:30.322842Z 2022-01-11T10:05:30.108042Z 2022-01-11T10:05:29.884058Z 2022-01-11T10:05:29.876446Z 2022-01-11T10:05:29.870107Z 2022-01-11T10:05:29.797060Z 2022-01-11T10:05:29.757387Z 2022-01-11T10:05:29.744653Z 2022-01-11T10:05:29.296306Z 2022-01-11T10:05:29.234372Z 2022-01-11T10:05:29.172284Z 2022-01-11T10:05:29.017371Z 2022-01-11T10:05:28.971559Z 2022-01-11T10:05:28.917983Z 2022-01-11T10:05:28.693233Z 2022-01-11T10:05:28.455200Z 2022-01-11T10:05:28.382083Z 2022-01-11T10:05:28.243446Z 2022-01-11T10:05:28.181339Z 2022-01-11T10:05:28.123932Z 2022-01-11T10:05:28.124299Z 2022-01-11T10:05:27.888181Z 2022-01-11T10:05:27.795885Z 2022-01-11T10:05:27.625577Z 2022-01-11T10:05:27.628841Z 2022-01-11T10:05:27.419971Z 2022-01-11T10:05:27.261337Z 2022-01-11T10:05:27.240632Z 2022-01-11T10:05:27.124578Z 2022-01-11T10:05:26.920009Z 2022-01-11T10:05:26.913786Z 2022-01-11T10:05:26.761176Z 2022-01-11T10:05:26.701420Z 2022-01-11T10:05:26.647533Z 2022-01-11T10:05:26.524409Z 2022-01-11T10:05:26.306297Z 2022-01-11T10:05:26.286203Z 2022-01-11T10:05:26.236527Z 2022-01-11T10:05:26.186538Z 2022-01-11T10:05:25.834141Z 2022-01-11T10:05:25.820553Z 2022-01-11T10:05:25.702811Z 2022-01-11T10:05:25.511327Z 2022-01-11T10:05:25.454660Z 2022-01-11T10:05:25.427127Z 2022-01-11T10:05:25.265049Z 2022-01-11T10:05:25.163817Z 2022-01-11T10:05:25.113742Z 2022-01-11T10:05:24.985019Z 2022-01-11T10:05:24.807256Z 2022-01-11T10:05:24.721633Z 2022-01-11T10:05:24.709713Z 2022-01-11T10:05:24.537664Z 2022-01-11T10:05:24.455982Z 2022-01-11T10:05:24.039268Z 2022-01-11T10:05:23.908709Z 2022-01-11T10:05:23.823368Z 2022-01-11T10:05:23.412084Z 2022-01-11T10:05:23.311005Z 2022-01-11T10:10:31.665293Z 2022-01-11T10:10:31.293859Z 2022-01-11T10:10:31.226995Z 2022-01-11T10:10:30.765956Z 2022-01-11T10:10:30.136798Z 2022-01-11T10:10:29.714318Z 2022-01-11T10:10:29.680542Z 2022-01-11T10:10:28.869107Z 2022-01-11T10:10:28.700623Z 2022-01-11T10:10:28.623468Z 2022-01-11T10:10:28.445458Z 2022-01-11T10:10:28.411348Z 2022-01-11T10:10:28.046476Z 2022-01-11T10:10:27.932954Z 2022-01-11T10:10:27.622992Z 2022-01-11T10:10:26.856528Z 2022-01-11T10:10:26.832954Z 2022-01-11T10:10:26.400437Z 2022-01-11T10:10:26.359333Z 2022-01-11T10:10:26.177537Z 2022-01-11T10:10:26.138052Z 2022-01-11T10:10:26.067332Z 2022-01-11T10:10:25.931931Z 2022-01-11T10:10:25.929436Z 2022-01-11T10:10:25.929294Z 2022-01-11T10:10:25.459297Z 2022-01-11T10:10:25.393366Z 2022-01-11T10:10:25.354376Z 2022-01-11T10:10:25.311302Z 2022-01-11T10:10:25.095880Z 2022-01-11T10:10:24.972452Z 2022-01-11T10:10:24.388729Z 2022-01-11T10:10:24.270797Z 2022-01-11T10:10:24.244585Z 2022-01-11T10:10:24.087031Z 2022-01-11T10:10:23.979835Z 2022-01-11T10:10:23.963363Z 2022-01-11T10:10:23.848115Z 2022-01-11T10:10:23.845544Z 2022-01-11T10:10:23.652403Z 2022-01-11T10:10:23.586905Z 2022-01-11T10:10:23.437667Z 2022-01-11T10:10:23.206342Z 2022-01-11T10:10:23.115768Z 2022-01-11T10:10:22.228541Z 2022-01-11T10:10:22.194489Z 2022-01-11T10:10:22.179630Z 2022-01-11T10:10:22.161727Z 2022-01-11T10:10:21.858747Z 2022-01-11T10:10:21.695798Z 2022-01-11T10:10:21.527147Z 2022-01-11T10:10:21.501354Z 2022-01-11T10:10:21.296884Z 2022-01-11T10:10:21.024524Z 2022-01-11T10:10:21.010992Z 2022-01-11T10:10:20.867910Z 2022-01-11T10:10:20.824301Z 2022-01-11T10:10:20.598317Z 2022-01-11T10:10:20.524425Z 2022-01-11T10:10:20.472053Z 2022-01-11T10:10:20.460692Z 2022-01-11T10:10:20.275986Z 2022-01-11T10:10:20.140674Z 2022-01-11T10:10:19.974048Z 2022-01-11T10:10:19.667536Z 2022-01-11T10:10:19.618810Z 2022-01-11T10:10:19.505490Z 2022-01-11T10:10:19.277156Z 2022-01-11T10:10:19.237118Z 2022-01-11T10:10:19.211444Z 2022-01-11T10:10:18.918407Z 2022-01-11T10:10:18.695387Z 2022-01-11T10:10:18.630829Z 2022-01-11T10:10:18.507262Z 2022-01-11T10:10:18.489326Z 2022-01-11T10:10:18.016308Z 2022-01-11T10:10:16.484010Z 2022-01-11T10:10:15.907210Z 2022-01-11T10:10:15.904654Z 2022-01-11T10:10:15.836233Z 2022-01-11T10:10:15.816616Z 2022-01-11T10:10:15.793245Z 2022-01-11T10:10:15.765291Z 2022-01-11T10:10:15.743105Z 2022-01-11T10:10:15.729669Z 2022-01-11T10:10:14.857839Z 2022-01-11T10:10:14.851307Z 2022-01-11T10:10:14.805652Z 2022-01-11T10:10:14.765133Z 2022-01-11T10:10:14.721334Z 2022-01-11T10:10:14.654517Z 2022-01-11T10:10:14.642618Z 2022-01-11T10:10:14.557320Z 2022-01-11T10:10:14.524104Z 2022-01-11T10:10:14.520067Z 2022-01-11T10:10:14.511248Z 2022-01-11T10:10:13.832403Z 2022-01-11T10:10:13.826886Z 2022-01-11T10:10:13.824947Z 2022-01-11T10:10:13.750111Z 2022-01-11T10:14:44.325810Z 2022-01-11T10:14:44.106606Z 2022-01-11T10:14:44.005311Z 2022-01-11T10:14:43.978979Z 2022-01-11T10:14:43.936011Z 2022-01-11T10:14:43.654588Z 2022-01-11T10:14:43.026620Z 2022-01-11T10:14:42.897460Z 2022-01-11T10:14:42.818551Z 2022-01-11T10:14:42.662340Z 2022-01-11T10:14:42.611477Z 2022-01-11T10:14:42.421823Z 2022-01-11T10:14:42.372345Z 2022-01-11T10:14:42.284642Z 2022-01-11T10:14:42.164529Z 2022-01-11T10:14:42.135790Z 2022-01-11T10:14:42.115076Z 2022-01-11T10:14:42.016535Z 2022-01-11T10:14:41.845097Z 2022-01-11T10:14:41.194500Z 2022-01-11T10:14:40.912671Z 2022-01-11T10:14:40.887887Z 2022-01-11T10:14:40.635700Z 2022-01-11T10:14:40.362440Z 2022-01-11T10:14:40.202532Z 2022-01-11T10:14:40.085865Z 2022-01-11T10:14:39.968959Z 2022-01-11T10:14:39.906829Z 2022-01-11T10:14:39.803480Z 2022-01-11T10:14:39.706843Z 2022-01-11T10:14:39.602658Z 2022-01-11T10:14:39.575328Z 2022-01-11T10:14:39.471401Z 2022-01-11T10:14:39.458973Z 2022-01-11T10:14:39.394455Z 2022-01-11T10:14:39.273243Z 2022-01-11T10:14:39.018657Z 2022-01-11T10:14:38.814441Z 2022-01-11T10:14:38.569810Z 2022-01-11T10:14:38.519563Z 2022-01-11T10:14:38.352002Z 2022-01-11T10:14:38.337687Z 2022-01-11T10:14:38.137898Z 2022-01-11T10:14:38.120673Z 2022-01-11T10:14:38.035630Z 2022-01-11T10:14:37.842737Z 2022-01-11T10:14:37.788188Z 2022-01-11T10:14:37.462425Z 2022-01-11T10:14:37.230332Z 2022-01-11T10:14:36.960459Z 2022-01-11T10:14:36.902894Z 2022-01-11T10:14:36.576974Z 2022-01-11T10:14:36.295385Z 2022-01-11T10:14:36.281723Z 2022-01-11T10:14:36.237828Z 2022-01-11T10:14:36.032993Z 2022-01-11T10:14:35.963461Z 2022-01-11T10:14:35.963209Z 2022-01-11T10:14:35.942269Z 2022-01-11T10:14:35.711892Z 2022-01-11T10:14:35.683312Z 2022-01-11T10:14:35.680086Z 2022-01-11T10:14:35.529987Z 2022-01-11T10:14:35.343630Z 2022-01-11T10:14:35.220486Z 2022-01-11T10:14:35.123912Z 2022-01-11T10:14:34.838741Z 2022-01-11T10:14:34.817454Z 2022-01-11T10:14:34.790568Z 2022-01-11T10:14:34.596532Z 2022-01-11T10:14:34.593496Z 2022-01-11T10:14:34.454993Z 2022-01-11T10:14:34.450675Z 2022-01-11T10:14:34.416934Z 2022-01-11T10:14:34.350052Z 2022-01-11T10:14:34.221019Z 2022-01-11T10:14:34.155630Z 2022-01-11T10:14:33.698924Z 2022-01-11T10:14:33.506339Z 2022-01-11T10:14:33.481312Z 2022-01-11T10:14:33.473373Z 2022-01-11T10:14:33.211745Z 2022-01-11T10:14:32.937440Z 2022-01-11T10:14:32.749714Z 2022-01-11T10:14:32.689864Z 2022-01-11T10:14:32.623972Z 2022-01-11T10:14:32.581136Z 2022-01-11T10:14:32.576974Z 2022-01-11T10:14:32.550013Z 2022-01-11T10:14:32.425251Z 2022-01-11T10:14:31.934308Z 2022-01-11T10:14:31.822923Z 2022-01-11T10:14:31.710713Z 2022-01-11T10:14:31.655741Z 2022-01-11T10:14:31.436058Z 2022-01-11T10:14:31.433061Z 2022-01-11T10:14:31.301558Z 2022-01-11T10:14:31.251247Z 2022-01-11T10:14:31.210929Z 2022-01-11T10:14:31.125998Z", + "created_diff": 18.6254664, + "ended": "2022-01-11T10:16:16.364735Z", + "error_job_ids": " []", + "event_first": "2022-01-11T09:58:07.171829Z 2022-01-11T09:57:40.810202Z 2022-01-11T09:58:01.231147Z 2022-01-11T09:57:38.879601Z 2022-01-11T09:57:59.789098Z 2022-01-11T09:57:40.062816Z 2022-01-11T09:58:04.900873Z 2022-01-11T09:57:29.992412Z 2022-01-11T09:58:09.207720Z 2022-01-11T09:57:33.621993Z 2022-01-11T09:58:07.423030Z 2022-01-11T09:57:45.504526Z 2022-01-11T09:58:06.750806Z 2022-01-11T09:57:29.248570Z 2022-01-11T09:57:54.983089Z 2022-01-11T09:58:35.691349Z 2022-01-11T09:57:49.431668Z 2022-01-11T09:58:18.506382Z 2022-01-11T09:57:45.484448Z 2022-01-11T09:59:03.692125Z 2022-01-11T09:57:32.112510Z 2022-01-11T09:57:54.163681Z 2022-01-11T09:57:20.996434Z 2022-01-11T09:58:27.368002Z 2022-01-11T09:57:11.098904Z 2022-01-11T09:59:13.224885Z 2022-01-11T09:57:09.402953Z 2022-01-11T09:58:45.287981Z 2022-01-11T09:57:05.081861Z 2022-01-11T09:59:23.449628Z 2022-01-11T09:57:01.038857Z 2022-01-11T09:58:55.480811Z 2022-01-11T09:56:57.252701Z 2022-01-11T09:57:04.005376Z 2022-01-11T09:56:56.002578Z 2022-01-11T09:58:35.491632Z 2022-01-11T09:58:16.673655Z 2022-01-11T09:58:34.067488Z 2022-01-11T09:58:36.854323Z 2022-01-11T09:58:33.034777Z 2022-01-11T09:58:12.250624Z 2022-01-11T09:58:33.290606Z 2022-01-11T09:58:15.283773Z 2022-01-11T09:58:34.552121Z 2022-01-11T09:58:12.495411Z 2022-01-11T09:58:29.266713Z 2022-01-11T09:58:13.767450Z 2022-01-11T09:58:30.258899Z 2022-01-11T09:58:15.787622Z 2022-01-11T09:58:29.777832Z 2022-01-11T09:58:11.306677Z 2022-01-11T09:58:26.578030Z 2022-01-11T09:58:12.495411Z 2022-01-11T09:58:32.534694Z 2022-01-11T09:58:12.495411Z 2022-01-11T09:58:30.049594Z 2022-01-11T09:58:11.563756Z 2022-01-11T09:58:29.505723Z 2022-01-11T09:58:13.335424Z 2022-01-11T09:58:31.518903Z 2022-01-11T09:58:09.791021Z 2022-01-11T09:58:31.693792Z 2022-01-11T09:58:02.149773Z 2022-01-11T09:58:19.318564Z 2022-01-11T09:57:59.654809Z 2022-01-11T09:58:18.309413Z 2022-01-11T09:58:00.156968Z 2022-01-11T09:58:17.704522Z 2022-01-11T09:57:54.675645Z 2022-01-11T09:58:26.080256Z 2022-01-11T09:58:17.290697Z 2022-01-11T09:58:22.980791Z 2022-01-11T09:58:00.666802Z 2022-01-11T09:58:19.061696Z 2022-01-11T09:58:04.957335Z 2022-01-11T09:58:27.108993Z 2022-01-11T09:58:20.153240Z 2022-01-11T09:58:01.186327Z 2022-01-11T09:58:21.948784Z 2022-01-11T09:58:01.185771Z 2022-01-11T09:58:25.740674Z 2022-01-11T09:58:02.149773Z 2022-01-11T09:58:23.436637Z 2022-01-11T09:57:51.644399Z 2022-01-11T09:58:19.335047Z 2022-01-11T09:57:54.420666Z 2022-01-11T09:58:10.373004Z 2022-01-11T09:57:57.278684Z 2022-01-11T09:58:07.420649Z 2022-01-11T09:57:52.124255Z 2022-01-11T09:58:09.022880Z 2022-01-11T09:57:49.084951Z 2022-01-11T09:58:04.119258Z 2022-01-11T09:57:53.669486Z 2022-01-11T09:58:16.005728Z 2022-01-11T09:57:54.423122Z 2022-01-11T09:58:10.626906Z 2022-01-11T09:57:48.767300Z 2022-01-11T09:58:08.321654Z 2022-01-11T09:57:40.306724Z 2022-01-11T10:03:01.752705Z 2022-01-11T10:03:01.565376Z 2022-01-11T10:03:02.863036Z 2022-01-11T10:03:02.535449Z 2022-01-11T10:03:02.871056Z 2022-01-11T10:03:02.804560Z 2022-01-11T10:02:58.239652Z 2022-01-11T10:02:58.483958Z 2022-01-11T10:03:03.517759Z 2022-01-11T10:03:02.794904Z 2022-01-11T10:02:58.239652Z 2022-01-11T10:02:58.033710Z 2022-01-11T10:03:02.863036Z 2022-01-11T10:02:59.710718Z 2022-01-11T10:03:02.614297Z 2022-01-11T10:02:58.425332Z 2022-01-11T10:02:58.239652Z 2022-01-11T10:02:55.167796Z 2022-01-11T10:02:53.568511Z 2022-01-11T10:03:00.199823Z 2022-01-11T10:02:59.249661Z 2022-01-11T10:02:55.871714Z 2022-01-11T10:02:57.983088Z 2022-01-11T10:02:55.754068Z 2022-01-11T10:02:59.076541Z 2022-01-11T10:02:58.934569Z 2022-01-11T10:02:56.178866Z 2022-01-11T10:02:59.935777Z 2022-01-11T10:02:54.118247Z 2022-01-11T10:02:56.519692Z 2022-01-11T10:02:58.239652Z 2022-01-11T10:02:58.425332Z 2022-01-11T10:02:56.979650Z 2022-01-11T10:02:58.006913Z 2022-01-11T10:02:47.011018Z 2022-01-11T10:02:58.499493Z 2022-01-11T10:02:53.811721Z 2022-01-11T10:02:53.787929Z 2022-01-11T10:02:46.422643Z 2022-01-11T10:02:50.068650Z 2022-01-11T10:02:48.228812Z 2022-01-11T10:02:45.742943Z 2022-01-11T10:02:39.964666Z 2022-01-11T10:02:51.369277Z 2022-01-11T10:02:44.648352Z 2022-01-11T10:02:51.462792Z 2022-01-11T10:02:43.974847Z 2022-01-11T10:02:46.004937Z 2022-01-11T10:02:44.698674Z 2022-01-11T10:02:46.504939Z 2022-01-11T10:02:40.214216Z 2022-01-11T10:02:47.528567Z 2022-01-11T10:02:40.745703Z 2022-01-11T10:02:45.238889Z 2022-01-11T10:02:40.820566Z 2022-01-11T10:02:33.479685Z 2022-01-11T10:02:31.684673Z 2022-01-11T10:02:37.648000Z 2022-01-11T10:02:36.179788Z 2022-01-11T10:02:39.158892Z 2022-01-11T10:02:35.925265Z 2022-01-11T10:02:41.749009Z 2022-01-11T10:02:34.178724Z 2022-01-11T10:02:32.472301Z 2022-01-11T10:02:22.962823Z 2022-01-11T10:02:36.627921Z 2022-01-11T10:02:29.565070Z 2022-01-11T10:02:30.809862Z 2022-01-11T10:02:28.553825Z 2022-01-11T10:02:28.706432Z 2022-01-11T10:02:27.765987Z 2022-01-11T10:02:36.127980Z 2022-01-11T10:02:21.827236Z 2022-01-11T10:02:32.472301Z 2022-01-11T10:02:15.358989Z 2022-01-11T10:02:22.479239Z 2022-01-11T10:02:11.950895Z 2022-01-11T10:02:21.448182Z 2022-01-11T10:02:10.478599Z 2022-01-11T10:01:50.902650Z 2022-01-11T10:02:25.349695Z 2022-01-11T10:02:13.176877Z 2022-01-11T10:02:21.190758Z 2022-01-11T10:01:49.914458Z 2022-01-11T10:01:44.890699Z 2022-01-11T10:01:54.404412Z 2022-01-11T10:01:46.658524Z 2022-01-11T10:01:50.499497Z 2022-01-11T10:01:44.142625Z 2022-01-11T10:01:48.631498Z 2022-01-11T10:01:45.284541Z 2022-01-11T10:01:42.740603Z 2022-01-11T10:01:42.361425Z 2022-01-11T10:01:45.650914Z 2022-01-11T10:01:36.194726Z 2022-01-11T10:01:37.914624Z 2022-01-11T10:01:32.279728Z 2022-01-11T10:01:39.715661Z 2022-01-11T10:01:30.840953Z 2022-01-11T10:01:32.861463Z 2022-01-11T10:07:04.436233Z 2022-01-11T10:07:01.650250Z 2022-01-11T10:07:04.979527Z 2022-01-11T10:07:01.565740Z 2022-01-11T10:06:59.886969Z 2022-01-11T10:07:00.580913Z 2022-01-11T10:07:01.979334Z 2022-01-11T10:06:59.093391Z 2022-01-11T10:07:05.240353Z 2022-01-11T10:06:58.595152Z 2022-01-11T10:06:56.531540Z 2022-01-11T10:06:57.698571Z 2022-01-11T10:07:03.234780Z 2022-01-11T10:07:00.410434Z 2022-01-11T10:06:58.115540Z 2022-01-11T10:06:54.919576Z 2022-01-11T10:06:59.251546Z 2022-01-11T10:06:59.761385Z 2022-01-11T10:06:56.382268Z 2022-01-11T10:06:52.324034Z 2022-01-11T10:07:03.284996Z 2022-01-11T10:06:55.546727Z 2022-01-11T10:06:46.434661Z 2022-01-11T10:06:58.384508Z 2022-01-11T10:07:03.725554Z 2022-01-11T10:06:57.123674Z 2022-01-11T10:06:52.570852Z 2022-01-11T10:06:55.694732Z 2022-01-11T10:07:00.879536Z 2022-01-11T10:06:54.138498Z 2022-01-11T10:06:59.753229Z 2022-01-11T10:06:55.499953Z 2022-01-11T10:06:47.550061Z 2022-01-11T10:06:55.597619Z 2022-01-11T10:06:58.993238Z 2022-01-11T10:06:51.809689Z 2022-01-11T10:06:52.576055Z 2022-01-11T10:06:46.555183Z 2022-01-11T10:06:50.614533Z 2022-01-11T10:06:50.134701Z 2022-01-11T10:06:53.335642Z 2022-01-11T10:06:53.079591Z 2022-01-11T10:06:55.373947Z 2022-01-11T10:06:53.579023Z 2022-01-11T10:06:48.084947Z 2022-01-11T10:06:46.883893Z 2022-01-11T10:06:49.086983Z 2022-01-11T10:06:51.892702Z 2022-01-11T10:06:48.090573Z 2022-01-11T10:06:47.173747Z 2022-01-11T10:06:43.456128Z 2022-01-11T10:06:40.657951Z 2022-01-11T10:06:49.086983Z 2022-01-11T10:06:41.819540Z 2022-01-11T10:06:55.373947Z 2022-01-11T10:06:44.253830Z 2022-01-11T10:06:49.338876Z 2022-01-11T10:06:39.784049Z 2022-01-11T10:06:50.410626Z 2022-01-11T10:06:45.260669Z 2022-01-11T10:06:41.819582Z 2022-01-11T10:06:48.334810Z 2022-01-11T10:06:40.025547Z 2022-01-11T10:06:43.772019Z 2022-01-11T10:06:37.029288Z 2022-01-11T10:06:43.906628Z 2022-01-11T10:06:16.911985Z 2022-01-11T10:06:24.375903Z 2022-01-11T10:06:42.434214Z 2022-01-11T10:06:13.063753Z 2022-01-11T10:06:24.699844Z 2022-01-11T10:06:15.692302Z 2022-01-11T10:06:11.475634Z 2022-01-11T10:06:21.242720Z 2022-01-11T10:06:11.869275Z 2022-01-11T10:05:58.741814Z 2022-01-11T10:06:12.358070Z 2022-01-11T10:06:22.814108Z 2022-01-11T10:06:01.804037Z 2022-01-11T10:06:11.399296Z 2022-01-11T10:06:06.137729Z 2022-01-11T10:05:59.045588Z 2022-01-11T10:06:06.378600Z 2022-01-11T10:06:00.259489Z 2022-01-11T10:06:02.400059Z 2022-01-11T10:05:58.191654Z 2022-01-11T10:05:54.440635Z 2022-01-11T10:05:55.227492Z 2022-01-11T10:05:54.647320Z 2022-01-11T10:06:02.065777Z 2022-01-11T10:05:54.940176Z 2022-01-11T10:05:45.257834Z 2022-01-11T10:05:41.148995Z 2022-01-11T10:05:44.688275Z 2022-01-11T10:05:40.661790Z 2022-01-11T10:05:40.498476Z 2022-01-11T10:05:37.078522Z 2022-01-11T10:05:35.841664Z 2022-01-11T10:05:31.774502Z 2022-01-11T10:05:32.179515Z 2022-01-11T10:11:53.033636Z 2022-01-11T10:11:55.379804Z 2022-01-11T10:11:49.382596Z 2022-01-11T10:11:50.734590Z 2022-01-11T10:11:53.933579Z 2022-01-11T10:11:52.450078Z 2022-01-11T10:11:55.140733Z 2022-01-11T10:11:51.168491Z 2022-01-11T10:11:50.816651Z 2022-01-11T10:11:43.580611Z 2022-01-11T10:11:52.394955Z 2022-01-11T10:11:46.852674Z 2022-01-11T10:11:50.568678Z 2022-01-11T10:11:51.168491Z 2022-01-11T10:11:51.067589Z 2022-01-11T10:11:47.864597Z 2022-01-11T10:11:51.960702Z 2022-01-11T10:11:49.382596Z 2022-01-11T10:11:53.933579Z 2022-01-11T10:11:50.748198Z 2022-01-11T10:11:49.628180Z 2022-01-11T10:11:43.580611Z 2022-01-11T10:11:50.568678Z 2022-01-11T10:11:45.779705Z 2022-01-11T10:11:51.388878Z 2022-01-11T10:11:44.134716Z 2022-01-11T10:11:46.877461Z 2022-01-11T10:11:44.594630Z 2022-01-11T10:11:49.346194Z 2022-01-11T10:11:45.107683Z 2022-01-11T10:11:46.305469Z 2022-01-11T10:11:40.520284Z 2022-01-11T10:11:45.968884Z 2022-01-11T10:11:42.565135Z 2022-01-11T10:11:48.032509Z 2022-01-11T10:11:40.063144Z 2022-01-11T10:11:43.172933Z 2022-01-11T10:11:39.670659Z 2022-01-11T10:11:43.667249Z 2022-01-11T10:11:41.038515Z 2022-01-11T10:11:49.346194Z 2022-01-11T10:11:41.897595Z 2022-01-11T10:11:40.929416Z 2022-01-11T10:11:37.189570Z 2022-01-11T10:11:38.532573Z 2022-01-11T10:11:36.864581Z 2022-01-11T10:11:42.432583Z 2022-01-11T10:11:14.865633Z 2022-01-11T10:11:44.174310Z 2022-01-11T10:11:35.210671Z 2022-01-11T10:11:38.902091Z 2022-01-11T10:11:29.577686Z 2022-01-11T10:11:35.615752Z 2022-01-11T10:11:33.196817Z 2022-01-11T10:11:38.532573Z 2022-01-11T10:11:35.237996Z 2022-01-11T10:11:38.164872Z 2022-01-11T10:11:27.605873Z 2022-01-11T10:11:31.598088Z 2022-01-11T10:11:23.994639Z 2022-01-11T10:11:33.949618Z 2022-01-11T10:11:22.427380Z 2022-01-11T10:11:36.036557Z 2022-01-11T10:11:12.914367Z 2022-01-11T10:11:26.186943Z 2022-01-11T10:11:25.722703Z 2022-01-11T10:11:27.122403Z 2022-01-11T10:11:16.123858Z 2022-01-11T10:11:35.144687Z 2022-01-11T10:11:22.394706Z 2022-01-11T10:10:49.512381Z 2022-01-11T10:10:59.144705Z 2022-01-11T10:10:48.581270Z 2022-01-11T10:10:52.456178Z 2022-01-11T10:10:52.700826Z 2022-01-11T10:10:58.882974Z 2022-01-11T10:10:47.986043Z 2022-01-11T10:10:55.630589Z 2022-01-11T10:10:46.119629Z 2022-01-11T10:10:52.840854Z 2022-01-11T10:10:47.220659Z 2022-01-11T10:10:54.863918Z 2022-01-11T10:10:44.200747Z 2022-01-11T10:10:49.812504Z 2022-01-11T10:10:46.874124Z 2022-01-11T10:10:40.146798Z 2022-01-11T10:10:36.598943Z 2022-01-11T10:10:42.310175Z 2022-01-11T10:10:39.192643Z 2022-01-11T10:10:38.086699Z 2022-01-11T10:10:34.714688Z 2022-01-11T10:10:37.219572Z 2022-01-11T10:10:34.670317Z 2022-01-11T10:10:35.256845Z 2022-01-11T10:10:32.759094Z 2022-01-11T10:10:34.927444Z 2022-01-11T10:10:30.326054Z 2022-01-11T10:10:27.213721Z 2022-01-11T10:10:27.037502Z 2022-01-11T10:10:26.736351Z 2022-01-11T10:16:14.411604Z 2022-01-11T10:16:09.144089Z 2022-01-11T10:16:15.380473Z 2022-01-11T10:16:09.394605Z 2022-01-11T10:16:12.750637Z 2022-01-11T10:16:08.888919Z 2022-01-11T10:16:15.371621Z 2022-01-11T10:16:08.658086Z 2022-01-11T10:16:12.504455Z 2022-01-11T10:16:07.975771Z 2022-01-11T10:16:07.034002Z 2022-01-11T10:16:03.990697Z 2022-01-11T10:16:10.529202Z 2022-01-11T10:16:07.903917Z 2022-01-11T10:16:11.126215Z 2022-01-11T10:16:08.639401Z 2022-01-11T10:16:15.336609Z 2022-01-11T10:16:07.658117Z 2022-01-11T10:16:11.232788Z 2022-01-11T10:16:04.415603Z 2022-01-11T10:16:09.100459Z 2022-01-11T10:16:01.817599Z 2022-01-11T10:16:07.210096Z 2022-01-11T10:16:04.614320Z 2022-01-11T10:16:11.126215Z 2022-01-11T10:16:01.956699Z 2022-01-11T10:16:09.100459Z 2022-01-11T10:16:04.929575Z 2022-01-11T10:16:07.105503Z 2022-01-11T10:16:03.253784Z 2022-01-11T10:16:03.795795Z 2022-01-11T10:16:02.532240Z 2022-01-11T10:16:06.732027Z 2022-01-11T10:16:02.792789Z 2022-01-11T10:16:09.500860Z 2022-01-11T10:16:01.025603Z 2022-01-11T10:16:05.835561Z 2022-01-11T10:16:00.777870Z 2022-01-11T10:16:01.021666Z 2022-01-11T10:15:59.254111Z 2022-01-11T10:15:53.158527Z 2022-01-11T10:15:56.976720Z 2022-01-11T10:16:00.776902Z 2022-01-11T10:15:51.552718Z 2022-01-11T10:16:04.305524Z 2022-01-11T10:15:54.925591Z 2022-01-11T10:16:06.534635Z 2022-01-11T10:15:56.754592Z 2022-01-11T10:15:58.499491Z 2022-01-11T10:15:52.916810Z 2022-01-11T10:16:04.794014Z 2022-01-11T10:15:53.661428Z 2022-01-11T10:15:53.770543Z 2022-01-11T10:15:46.601671Z 2022-01-11T10:15:59.595632Z 2022-01-11T10:15:43.198640Z 2022-01-11T10:15:59.075639Z 2022-01-11T10:15:40.978927Z 2022-01-11T10:15:49.230675Z 2022-01-11T10:15:49.291725Z 2022-01-11T10:15:49.758576Z 2022-01-11T10:15:44.435526Z 2022-01-11T10:15:50.459235Z 2022-01-11T10:15:34.783240Z 2022-01-11T10:15:44.300032Z 2022-01-11T10:15:27.277447Z 2022-01-11T10:15:48.393242Z 2022-01-11T10:15:33.633900Z 2022-01-11T10:15:45.392771Z 2022-01-11T10:15:32.235167Z 2022-01-11T10:15:39.211871Z 2022-01-11T10:15:27.308685Z 2022-01-11T10:15:39.864663Z 2022-01-11T10:15:08.381678Z 2022-01-11T10:15:27.758381Z 2022-01-11T10:15:36.416318Z 2022-01-11T10:15:21.026033Z 2022-01-11T10:14:59.076652Z 2022-01-11T10:15:04.383551Z 2022-01-11T10:14:58.503775Z 2022-01-11T10:15:05.591552Z 2022-01-11T10:14:58.503775Z 2022-01-11T10:15:01.674731Z 2022-01-11T10:14:58.126615Z 2022-01-11T10:15:03.194325Z 2022-01-11T10:14:59.213699Z 2022-01-11T10:15:00.250763Z 2022-01-11T10:14:55.821855Z 2022-01-11T10:14:58.562896Z 2022-01-11T10:14:56.328626Z 2022-01-11T10:14:51.464034Z 2022-01-11T10:14:51.675923Z 2022-01-11T10:14:52.292197Z 2022-01-11T10:14:50.915696Z 2022-01-11T10:14:50.215077Z 2022-01-11T10:14:49.695456Z 2022-01-11T10:14:47.799493Z 2022-01-11T10:14:43.294694Z 2022-01-11T10:14:41.975503Z 2022-01-11T10:14:41.949940Z", + "event_last": "2022-01-11T09:58:13.170414Z 2022-01-11T09:57:46.240370Z 2022-01-11T09:58:06.758530Z 2022-01-11T09:57:43.736267Z 2022-01-11T09:58:03.367897Z 2022-01-11T09:57:45.219424Z 2022-01-11T09:58:08.507438Z 2022-01-11T09:57:34.631437Z 2022-01-11T09:58:15.642622Z 2022-01-11T09:57:38.354379Z 2022-01-11T09:58:12.716411Z 2022-01-11T09:57:49.084951Z 2022-01-11T09:58:11.656173Z 2022-01-11T09:57:34.287428Z 2022-01-11T09:57:59.070421Z 2022-01-11T09:58:36.353866Z 2022-01-11T09:57:52.796534Z 2022-01-11T09:58:20.676924Z 2022-01-11T09:57:50.338859Z 2022-01-11T09:59:05.412021Z 2022-01-11T09:57:37.109387Z 2022-01-11T09:57:57.508522Z 2022-01-11T09:57:23.292558Z 2022-01-11T09:58:29.501998Z 2022-01-11T09:57:13.312450Z 2022-01-11T09:59:14.173151Z 2022-01-11T09:57:12.057071Z 2022-01-11T09:58:45.687095Z 2022-01-11T09:57:07.211805Z 2022-01-11T09:59:26.115769Z 2022-01-11T09:57:03.149284Z 2022-01-11T09:58:55.645376Z 2022-01-11T09:56:58.591931Z 2022-01-11T09:57:05.465014Z 2022-01-11T09:56:57.541560Z 2022-01-11T09:58:36.457783Z 2022-01-11T09:58:17.354130Z 2022-01-11T09:58:35.571487Z 2022-01-11T09:58:37.292963Z 2022-01-11T09:58:35.192981Z 2022-01-11T09:58:14.584430Z 2022-01-11T09:58:35.372295Z 2022-01-11T09:58:16.348648Z 2022-01-11T09:58:36.089436Z 2022-01-11T09:58:14.770828Z 2022-01-11T09:58:33.251474Z 2022-01-11T09:58:15.824369Z 2022-01-11T09:58:33.165160Z 2022-01-11T09:58:16.675486Z 2022-01-11T09:58:32.134252Z 2022-01-11T09:58:13.869403Z 2022-01-11T09:58:28.603438Z 2022-01-11T09:58:14.464202Z 2022-01-11T09:58:34.814418Z 2022-01-11T09:58:14.474294Z 2022-01-11T09:58:32.795984Z 2022-01-11T09:58:13.665917Z 2022-01-11T09:58:32.223885Z 2022-01-11T09:58:15.362532Z 2022-01-11T09:58:34.067488Z 2022-01-11T09:58:11.880366Z 2022-01-11T09:58:33.945703Z 2022-01-11T09:58:05.792661Z 2022-01-11T09:58:24.393972Z 2022-01-11T09:58:01.879610Z 2022-01-11T09:58:22.937577Z 2022-01-11T09:58:03.080671Z 2022-01-11T09:58:22.389557Z 2022-01-11T09:57:57.816992Z 2022-01-11T09:58:28.866544Z 2022-01-11T09:58:21.485508Z 2022-01-11T09:58:26.567047Z 2022-01-11T09:58:03.330378Z 2022-01-11T09:58:24.824390Z 2022-01-11T09:58:07.561403Z 2022-01-11T09:58:29.289188Z 2022-01-11T09:58:25.240951Z 2022-01-11T09:58:04.897289Z 2022-01-11T09:58:23.745485Z 2022-01-11T09:58:04.964473Z 2022-01-11T09:58:28.866544Z 2022-01-11T09:58:04.881389Z 2022-01-11T09:58:26.336234Z 2022-01-11T09:57:54.417347Z 2022-01-11T09:58:24.232329Z 2022-01-11T09:57:57.249364Z 2022-01-11T09:58:15.160413Z 2022-01-11T09:57:59.909685Z 2022-01-11T09:58:13.145445Z 2022-01-11T09:57:55.750324Z 2022-01-11T09:58:14.749931Z 2022-01-11T09:57:53.669486Z 2022-01-11T09:58:07.677455Z 2022-01-11T09:57:56.975424Z 2022-01-11T09:58:21.247591Z 2022-01-11T09:57:58.128385Z 2022-01-11T09:58:14.835218Z 2022-01-11T09:57:53.304955Z 2022-01-11T09:58:12.927988Z 2022-01-11T09:57:44.119247Z 2022-01-11T10:03:03.263528Z 2022-01-11T10:03:03.469264Z 2022-01-11T10:03:04.256704Z 2022-01-11T10:03:03.636766Z 2022-01-11T10:03:04.067607Z 2022-01-11T10:03:05.947905Z 2022-01-11T10:03:00.863510Z 2022-01-11T10:03:01.377470Z 2022-01-11T10:03:04.467433Z 2022-01-11T10:03:05.947905Z 2022-01-11T10:03:00.498398Z 2022-01-11T10:03:00.636147Z 2022-01-11T10:03:04.067607Z 2022-01-11T10:03:01.682037Z 2022-01-11T10:03:04.067607Z 2022-01-11T10:03:01.343628Z 2022-01-11T10:03:01.578488Z 2022-01-11T10:02:58.268548Z 2022-01-11T10:02:56.542422Z 2022-01-11T10:03:02.623814Z 2022-01-11T10:03:01.578488Z 2022-01-11T10:02:59.088420Z 2022-01-11T10:03:00.665664Z 2022-01-11T10:02:58.438515Z 2022-01-11T10:03:02.199065Z 2022-01-11T10:03:01.572796Z 2022-01-11T10:02:58.553344Z 2022-01-11T10:03:02.535449Z 2022-01-11T10:02:56.694463Z 2022-01-11T10:02:59.515384Z 2022-01-11T10:03:00.856441Z 2022-01-11T10:03:01.006512Z 2022-01-11T10:02:59.490203Z 2022-01-11T10:03:00.409997Z 2022-01-11T10:02:51.983444Z 2022-01-11T10:03:01.173396Z 2022-01-11T10:02:55.974503Z 2022-01-11T10:02:56.356451Z 2022-01-11T10:02:49.453428Z 2022-01-11T10:02:52.877587Z 2022-01-11T10:02:51.228384Z 2022-01-11T10:02:49.627289Z 2022-01-11T10:02:43.066395Z 2022-01-11T10:02:54.882440Z 2022-01-11T10:02:49.367926Z 2022-01-11T10:02:54.529044Z 2022-01-11T10:02:48.016466Z 2022-01-11T10:02:50.457925Z 2022-01-11T10:02:48.269503Z 2022-01-11T10:02:50.392405Z 2022-01-11T10:02:44.018553Z 2022-01-11T10:02:51.420384Z 2022-01-11T10:02:44.030398Z 2022-01-11T10:02:49.285161Z 2022-01-11T10:02:44.239400Z 2022-01-11T10:02:36.936640Z 2022-01-11T10:02:35.597614Z 2022-01-11T10:02:41.722397Z 2022-01-11T10:02:40.476412Z 2022-01-11T10:02:44.065879Z 2022-01-11T10:02:40.042456Z 2022-01-11T10:02:45.835437Z 2022-01-11T10:02:38.520403Z 2022-01-11T10:02:37.727912Z 2022-01-11T10:02:27.726481Z 2022-01-11T10:02:41.362261Z 2022-01-11T10:02:32.485348Z 2022-01-11T10:02:35.531520Z 2022-01-11T10:02:32.755476Z 2022-01-11T10:02:32.864596Z 2022-01-11T10:02:31.673386Z 2022-01-11T10:02:41.189393Z 2022-01-11T10:02:25.670176Z 2022-01-11T10:02:35.972716Z 2022-01-11T10:02:19.304921Z 2022-01-11T10:02:26.952483Z 2022-01-11T10:02:18.167778Z 2022-01-11T10:02:25.648482Z 2022-01-11T10:02:15.016221Z 2022-01-11T10:01:53.176130Z 2022-01-11T10:02:28.974427Z 2022-01-11T10:02:18.955439Z 2022-01-11T10:02:26.026662Z 2022-01-11T10:01:52.531436Z 2022-01-11T10:01:46.492392Z 2022-01-11T10:01:57.628834Z 2022-01-11T10:01:48.565540Z 2022-01-11T10:01:53.316551Z 2022-01-11T10:01:45.470391Z 2022-01-11T10:01:50.807967Z 2022-01-11T10:01:47.231221Z 2022-01-11T10:01:45.660507Z 2022-01-11T10:01:43.680368Z 2022-01-11T10:01:47.567760Z 2022-01-11T10:01:37.675631Z 2022-01-11T10:01:39.188618Z 2022-01-11T10:01:34.054779Z 2022-01-11T10:01:40.685508Z 2022-01-11T10:01:32.351865Z 2022-01-11T10:01:34.366002Z 2022-01-11T10:07:06.006481Z 2022-01-11T10:07:03.126362Z 2022-01-11T10:07:06.219015Z 2022-01-11T10:07:05.415841Z 2022-01-11T10:07:02.747895Z 2022-01-11T10:07:02.551427Z 2022-01-11T10:07:04.048204Z 2022-01-11T10:07:01.531052Z 2022-01-11T10:07:06.005417Z 2022-01-11T10:07:01.419033Z 2022-01-11T10:06:59.953373Z 2022-01-11T10:07:00.410434Z 2022-01-11T10:07:04.564357Z 2022-01-11T10:07:02.880554Z 2022-01-11T10:07:00.680408Z 2022-01-11T10:06:57.564602Z 2022-01-11T10:07:01.627706Z 2022-01-11T10:07:01.929521Z 2022-01-11T10:06:59.264840Z 2022-01-11T10:06:55.914465Z 2022-01-11T10:07:04.899393Z 2022-01-11T10:06:58.431225Z 2022-01-11T10:06:50.237578Z 2022-01-11T10:07:01.314561Z 2022-01-11T10:07:05.316583Z 2022-01-11T10:07:00.249689Z 2022-01-11T10:06:57.284271Z 2022-01-11T10:06:59.552637Z 2022-01-11T10:07:03.300770Z 2022-01-11T10:06:56.721451Z 2022-01-11T10:07:01.861363Z 2022-01-11T10:06:59.054464Z 2022-01-11T10:06:51.274707Z 2022-01-11T10:06:59.451305Z 2022-01-11T10:07:02.042898Z 2022-01-11T10:06:55.377446Z 2022-01-11T10:06:56.264395Z 2022-01-11T10:06:51.083480Z 2022-01-11T10:06:54.883057Z 2022-01-11T10:06:52.985725Z 2022-01-11T10:06:57.464525Z 2022-01-11T10:06:56.411527Z 2022-01-11T10:06:58.893354Z 2022-01-11T10:06:56.888093Z 2022-01-11T10:06:51.580167Z 2022-01-11T10:06:50.433462Z 2022-01-11T10:06:53.471417Z 2022-01-11T10:06:55.724114Z 2022-01-11T10:06:52.402927Z 2022-01-11T10:06:50.910730Z 2022-01-11T10:06:46.982511Z 2022-01-11T10:06:43.402521Z 2022-01-11T10:06:53.609205Z 2022-01-11T10:06:45.971628Z 2022-01-11T10:06:58.659031Z 2022-01-11T10:06:47.849696Z 2022-01-11T10:06:53.609205Z 2022-01-11T10:06:43.128408Z 2022-01-11T10:06:54.264115Z 2022-01-11T10:06:49.397883Z 2022-01-11T10:06:45.869861Z 2022-01-11T10:06:53.922452Z 2022-01-11T10:06:43.093841Z 2022-01-11T10:06:47.193924Z 2022-01-11T10:06:39.787188Z 2022-01-11T10:06:47.187363Z 2022-01-11T10:06:20.409553Z 2022-01-11T10:06:28.897012Z 2022-01-11T10:06:45.951479Z 2022-01-11T10:06:17.529404Z 2022-01-11T10:06:29.299022Z 2022-01-11T10:06:20.955462Z 2022-01-11T10:06:18.080477Z 2022-01-11T10:06:25.103520Z 2022-01-11T10:06:17.165932Z 2022-01-11T10:06:01.973793Z 2022-01-11T10:06:18.437610Z 2022-01-11T10:06:26.113463Z 2022-01-11T10:06:06.198817Z 2022-01-11T10:06:15.524963Z 2022-01-11T10:06:11.506810Z 2022-01-11T10:06:02.002763Z 2022-01-11T10:06:11.874529Z 2022-01-11T10:06:05.314021Z 2022-01-11T10:06:06.130441Z 2022-01-11T10:06:02.007636Z 2022-01-11T10:05:57.122745Z 2022-01-11T10:05:57.692403Z 2022-01-11T10:05:58.634760Z 2022-01-11T10:06:06.495842Z 2022-01-11T10:05:57.117418Z 2022-01-11T10:05:46.757467Z 2022-01-11T10:05:42.755625Z 2022-01-11T10:05:45.630017Z 2022-01-11T10:05:42.723349Z 2022-01-11T10:05:43.598518Z 2022-01-11T10:05:38.373057Z 2022-01-11T10:05:38.916634Z 2022-01-11T10:05:32.911286Z 2022-01-11T10:05:33.583310Z 2022-01-11T10:11:56.320411Z 2022-01-11T10:11:57.821651Z 2022-01-11T10:11:51.113057Z 2022-01-11T10:11:52.813181Z 2022-01-11T10:11:55.540408Z 2022-01-11T10:11:53.850381Z 2022-01-11T10:11:56.250578Z 2022-01-11T10:11:53.069216Z 2022-01-11T10:11:53.611846Z 2022-01-11T10:11:47.525702Z 2022-01-11T10:11:54.591535Z 2022-01-11T10:11:49.518961Z 2022-01-11T10:11:53.334421Z 2022-01-11T10:11:52.813181Z 2022-01-11T10:11:53.843739Z 2022-01-11T10:11:50.667200Z 2022-01-11T10:11:54.205441Z 2022-01-11T10:11:51.310695Z 2022-01-11T10:11:55.923912Z 2022-01-11T10:11:52.813181Z 2022-01-11T10:11:52.340025Z 2022-01-11T10:11:47.425266Z 2022-01-11T10:11:53.527414Z 2022-01-11T10:11:48.215436Z 2022-01-11T10:11:54.064804Z 2022-01-11T10:11:46.865773Z 2022-01-11T10:11:49.833540Z 2022-01-11T10:11:47.525702Z 2022-01-11T10:11:51.515422Z 2022-01-11T10:11:47.985844Z 2022-01-11T10:11:50.137274Z 2022-01-11T10:11:43.147395Z 2022-01-11T10:11:49.114518Z 2022-01-11T10:11:45.336708Z 2022-01-11T10:11:51.242407Z 2022-01-11T10:11:43.190890Z 2022-01-11T10:11:46.296990Z 2022-01-11T10:11:43.460351Z 2022-01-11T10:11:47.072592Z 2022-01-11T10:11:44.206943Z 2022-01-11T10:11:51.561642Z 2022-01-11T10:11:43.608833Z 2022-01-11T10:11:45.339449Z 2022-01-11T10:11:40.569535Z 2022-01-11T10:11:42.345455Z 2022-01-11T10:11:39.575305Z 2022-01-11T10:11:46.104923Z 2022-01-11T10:11:18.360357Z 2022-01-11T10:11:47.072592Z 2022-01-11T10:11:38.327398Z 2022-01-11T10:11:43.012993Z 2022-01-11T10:11:33.014414Z 2022-01-11T10:11:39.537587Z 2022-01-11T10:11:35.773367Z 2022-01-11T10:11:42.925769Z 2022-01-11T10:11:38.148458Z 2022-01-11T10:11:41.437177Z 2022-01-11T10:11:30.826426Z 2022-01-11T10:11:34.509419Z 2022-01-11T10:11:28.482441Z 2022-01-11T10:11:37.341448Z 2022-01-11T10:11:26.205356Z 2022-01-11T10:11:40.130040Z 2022-01-11T10:11:16.383801Z 2022-01-11T10:11:31.068489Z 2022-01-11T10:11:29.378542Z 2022-01-11T10:11:31.235529Z 2022-01-11T10:11:18.846488Z 2022-01-11T10:11:39.277430Z 2022-01-11T10:11:26.175428Z 2022-01-11T10:10:52.223454Z 2022-01-11T10:11:04.623454Z 2022-01-11T10:10:51.683667Z 2022-01-11T10:10:57.164696Z 2022-01-11T10:10:56.917870Z 2022-01-11T10:11:03.412939Z 2022-01-11T10:10:50.597544Z 2022-01-11T10:11:00.393121Z 2022-01-11T10:10:47.879470Z 2022-01-11T10:10:56.277473Z 2022-01-11T10:10:49.964246Z 2022-01-11T10:10:58.718460Z 2022-01-11T10:10:45.557359Z 2022-01-11T10:10:52.498377Z 2022-01-11T10:10:49.257085Z 2022-01-11T10:10:41.214642Z 2022-01-11T10:10:37.627976Z 2022-01-11T10:10:43.580150Z 2022-01-11T10:10:40.463381Z 2022-01-11T10:10:39.592457Z 2022-01-11T10:10:36.639318Z 2022-01-11T10:10:39.026416Z 2022-01-11T10:10:36.492376Z 2022-01-11T10:10:37.219572Z 2022-01-11T10:10:34.160589Z 2022-01-11T10:10:36.702299Z 2022-01-11T10:10:32.276091Z 2022-01-11T10:10:29.645936Z 2022-01-11T10:10:30.328272Z 2022-01-11T10:10:29.358265Z 2022-01-11T10:16:15.489376Z 2022-01-11T10:16:10.857733Z 2022-01-11T10:16:16.364735Z 2022-01-11T10:16:12.555592Z 2022-01-11T10:16:14.621675Z 2022-01-11T10:16:10.429085Z 2022-01-11T10:16:16.353714Z 2022-01-11T10:16:10.711518Z 2022-01-11T10:16:14.335879Z 2022-01-11T10:16:09.829518Z 2022-01-11T10:16:11.039213Z 2022-01-11T10:16:06.297853Z 2022-01-11T10:16:12.802215Z 2022-01-11T10:16:09.423516Z 2022-01-11T10:16:13.712997Z 2022-01-11T10:16:10.857733Z 2022-01-11T10:16:16.300026Z 2022-01-11T10:16:09.346266Z 2022-01-11T10:16:13.940673Z 2022-01-11T10:16:07.845814Z 2022-01-11T10:16:12.923485Z 2022-01-11T10:16:04.964131Z 2022-01-11T10:16:10.171827Z 2022-01-11T10:16:07.090735Z 2022-01-11T10:16:13.797376Z 2022-01-11T10:16:05.212079Z 2022-01-11T10:16:11.674419Z 2022-01-11T10:16:07.746427Z 2022-01-11T10:16:11.224186Z 2022-01-11T10:16:06.635711Z 2022-01-11T10:16:06.671640Z 2022-01-11T10:16:05.344535Z 2022-01-11T10:16:10.171827Z 2022-01-11T10:16:05.676625Z 2022-01-11T10:16:12.310416Z 2022-01-11T10:16:04.324283Z 2022-01-11T10:16:09.048429Z 2022-01-11T10:16:03.788611Z 2022-01-11T10:16:05.260403Z 2022-01-11T10:16:02.025274Z 2022-01-11T10:15:57.258653Z 2022-01-11T10:16:00.239384Z 2022-01-11T10:16:04.168370Z 2022-01-11T10:15:54.071453Z 2022-01-11T10:16:07.726642Z 2022-01-11T10:15:57.820587Z 2022-01-11T10:16:10.736112Z 2022-01-11T10:15:59.470074Z 2022-01-11T10:16:02.130101Z 2022-01-11T10:15:56.699535Z 2022-01-11T10:16:08.535047Z 2022-01-11T10:15:57.226413Z 2022-01-11T10:15:57.857508Z 2022-01-11T10:15:49.829182Z 2022-01-11T10:16:03.414376Z 2022-01-11T10:15:46.961498Z 2022-01-11T10:16:02.396450Z 2022-01-11T10:15:44.315934Z 2022-01-11T10:15:53.785576Z 2022-01-11T10:15:51.905427Z 2022-01-11T10:15:53.597377Z 2022-01-11T10:15:48.298494Z 2022-01-11T10:15:54.314483Z 2022-01-11T10:15:38.916810Z 2022-01-11T10:15:47.290072Z 2022-01-11T10:15:31.039478Z 2022-01-11T10:15:52.969500Z 2022-01-11T10:15:37.402494Z 2022-01-11T10:15:49.846450Z 2022-01-11T10:15:34.511604Z 2022-01-11T10:15:42.977435Z 2022-01-11T10:15:31.173523Z 2022-01-11T10:15:44.079363Z 2022-01-11T10:15:10.531437Z 2022-01-11T10:15:31.868974Z 2022-01-11T10:15:40.990390Z 2022-01-11T10:15:25.915449Z 2022-01-11T10:15:00.644339Z 2022-01-11T10:15:07.292985Z 2022-01-11T10:15:00.644339Z 2022-01-11T10:15:09.012183Z 2022-01-11T10:15:00.710076Z 2022-01-11T10:15:04.189338Z 2022-01-11T10:14:59.755460Z 2022-01-11T10:15:05.365412Z 2022-01-11T10:15:00.977364Z 2022-01-11T10:15:02.765310Z 2022-01-11T10:14:56.850554Z 2022-01-11T10:14:59.949734Z 2022-01-11T10:14:57.376772Z 2022-01-11T10:14:52.582668Z 2022-01-11T10:14:52.579465Z 2022-01-11T10:14:55.761105Z 2022-01-11T10:14:52.421096Z 2022-01-11T10:14:51.695347Z 2022-01-11T10:14:50.977479Z 2022-01-11T10:14:49.186770Z 2022-01-11T10:14:44.503772Z 2022-01-11T10:14:43.753385Z 2022-01-11T10:14:43.781145Z", + "failed_job_ids": " []", + "finished": "2022-01-11T09:58:18.126192Z 2022-01-11T09:57:51.578725Z 2022-01-11T09:58:12.605059Z 2022-01-11T09:57:49.276092Z 2022-01-11T09:58:10.455163Z 2022-01-11T09:57:49.842309Z 2022-01-11T09:58:15.267278Z 2022-01-11T09:57:40.966546Z 2022-01-11T09:58:20.827037Z 2022-01-11T09:57:42.408877Z 2022-01-11T09:58:19.566438Z 2022-01-11T09:57:54.268709Z 2022-01-11T09:58:18.082989Z 2022-01-11T09:57:39.630494Z 2022-01-11T09:58:04.598042Z 2022-01-11T09:58:40.042291Z 2022-01-11T09:57:58.774394Z 2022-01-11T09:58:23.418874Z 2022-01-11T09:57:55.342260Z 2022-01-11T09:59:08.016850Z 2022-01-11T09:57:43.269084Z 2022-01-11T09:58:03.160523Z 2022-01-11T09:57:29.153353Z 2022-01-11T09:58:31.450997Z 2022-01-11T09:57:18.409408Z 2022-01-11T09:59:17.799873Z 2022-01-11T09:57:15.514521Z 2022-01-11T09:58:49.622573Z 2022-01-11T09:57:11.830673Z 2022-01-11T09:59:28.034003Z 2022-01-11T09:57:07.447122Z 2022-01-11T09:58:59.574726Z 2022-01-11T09:57:02.687493Z 2022-01-11T09:57:09.459424Z 2022-01-11T09:57:01.937279Z 2022-01-11T09:58:40.517985Z 2022-01-11T09:58:21.147486Z 2022-01-11T09:58:39.986556Z 2022-01-11T09:58:41.355791Z 2022-01-11T09:58:39.713829Z 2022-01-11T09:58:18.897094Z 2022-01-11T09:58:40.105717Z 2022-01-11T09:58:20.091958Z 2022-01-11T09:58:40.318867Z 2022-01-11T09:58:19.149643Z 2022-01-11T09:58:38.764010Z 2022-01-11T09:58:20.343114Z 2022-01-11T09:58:38.504534Z 2022-01-11T09:58:20.920673Z 2022-01-11T09:58:36.535707Z 2022-01-11T09:58:17.782668Z 2022-01-11T09:58:34.232428Z 2022-01-11T09:58:19.148507Z 2022-01-11T09:58:39.027364Z 2022-01-11T09:58:19.314651Z 2022-01-11T09:58:39.025410Z 2022-01-11T09:58:18.286257Z 2022-01-11T09:58:36.885959Z 2022-01-11T09:58:20.172425Z 2022-01-11T09:58:39.032070Z 2022-01-11T09:58:16.498181Z 2022-01-11T09:58:38.550871Z 2022-01-11T09:58:11.097177Z 2022-01-11T09:58:29.957985Z 2022-01-11T09:58:06.848589Z 2022-01-11T09:58:28.044551Z 2022-01-11T09:58:08.760409Z 2022-01-11T09:58:28.224865Z 2022-01-11T09:58:03.103221Z 2022-01-11T09:58:34.752388Z 2022-01-11T09:58:26.464991Z 2022-01-11T09:58:31.388299Z 2022-01-11T09:58:08.259348Z 2022-01-11T09:58:30.564722Z 2022-01-11T09:58:12.219138Z 2022-01-11T09:58:34.832405Z 2022-01-11T09:58:29.415875Z 2022-01-11T09:58:08.910295Z 2022-01-11T09:58:29.038846Z 2022-01-11T09:58:10.978509Z 2022-01-11T09:58:33.835199Z 2022-01-11T09:58:09.345352Z 2022-01-11T09:58:30.952397Z 2022-01-11T09:58:00.853351Z 2022-01-11T09:58:29.494181Z 2022-01-11T09:58:02.991323Z 2022-01-11T09:58:19.671732Z 2022-01-11T09:58:05.725303Z 2022-01-11T09:58:19.854325Z 2022-01-11T09:58:02.127318Z 2022-01-11T09:58:19.499256Z 2022-01-11T09:58:00.342313Z 2022-01-11T09:58:13.391280Z 2022-01-11T09:58:02.659318Z 2022-01-11T09:58:28.177236Z 2022-01-11T09:58:02.648263Z 2022-01-11T09:58:21.715115Z 2022-01-11T09:57:59.695928Z 2022-01-11T09:58:19.197284Z 2022-01-11T09:57:50.194119Z 2022-01-11T10:03:07.701400Z 2022-01-11T10:03:07.468531Z 2022-01-11T10:03:07.954506Z 2022-01-11T10:03:07.434459Z 2022-01-11T10:03:07.931997Z 2022-01-11T10:03:07.600430Z 2022-01-11T10:03:05.517700Z 2022-01-11T10:03:06.341363Z 2022-01-11T10:03:08.728861Z 2022-01-11T10:03:07.902837Z 2022-01-11T10:03:04.921369Z 2022-01-11T10:03:05.650970Z 2022-01-11T10:03:08.537810Z 2022-01-11T10:03:06.334426Z 2022-01-11T10:03:07.737310Z 2022-01-11T10:03:05.686338Z 2022-01-11T10:03:06.231099Z 2022-01-11T10:03:02.410556Z 2022-01-11T10:03:01.411534Z 2022-01-11T10:03:07.433509Z 2022-01-11T10:03:06.269734Z 2022-01-11T10:03:04.141253Z 2022-01-11T10:03:04.945020Z 2022-01-11T10:03:02.670822Z 2022-01-11T10:03:06.344981Z 2022-01-11T10:03:06.462932Z 2022-01-11T10:03:02.770345Z 2022-01-11T10:03:06.621660Z 2022-01-11T10:03:01.168910Z 2022-01-11T10:03:04.188617Z 2022-01-11T10:03:05.081883Z 2022-01-11T10:03:05.930918Z 2022-01-11T10:03:04.820015Z 2022-01-11T10:03:04.419690Z 2022-01-11T10:02:55.991785Z 2022-01-11T10:03:06.031666Z 2022-01-11T10:02:59.882817Z 2022-01-11T10:03:00.950987Z 2022-01-11T10:02:54.018860Z 2022-01-11T10:02:58.716141Z 2022-01-11T10:02:56.836324Z 2022-01-11T10:02:55.054649Z 2022-01-11T10:02:48.261306Z 2022-01-11T10:03:00.079860Z 2022-01-11T10:02:53.934628Z 2022-01-11T10:02:59.003136Z 2022-01-11T10:02:53.551998Z 2022-01-11T10:02:55.333242Z 2022-01-11T10:02:53.889274Z 2022-01-11T10:02:55.651270Z 2022-01-11T10:02:50.282363Z 2022-01-11T10:02:56.565678Z 2022-01-11T10:02:49.205991Z 2022-01-11T10:02:54.111133Z 2022-01-11T10:02:48.529276Z 2022-01-11T10:02:42.590804Z 2022-01-11T10:02:40.556957Z 2022-01-11T10:02:47.711964Z 2022-01-11T10:02:45.989729Z 2022-01-11T10:02:49.183889Z 2022-01-11T10:02:45.069867Z 2022-01-11T10:02:51.666362Z 2022-01-11T10:02:43.656045Z 2022-01-11T10:02:43.722145Z 2022-01-11T10:02:33.743255Z 2022-01-11T10:02:46.708565Z 2022-01-11T10:02:38.892745Z 2022-01-11T10:02:41.043408Z 2022-01-11T10:02:37.020406Z 2022-01-11T10:02:39.457494Z 2022-01-11T10:02:38.152081Z 2022-01-11T10:02:47.346257Z 2022-01-11T10:02:29.991351Z 2022-01-11T10:02:42.064137Z 2022-01-11T10:02:25.452341Z 2022-01-11T10:02:33.003650Z 2022-01-11T10:02:23.990191Z 2022-01-11T10:02:31.381038Z 2022-01-11T10:02:21.648612Z 2022-01-11T10:01:58.130223Z 2022-01-11T10:02:34.667225Z 2022-01-11T10:02:25.276318Z 2022-01-11T10:02:32.045493Z 2022-01-11T10:01:56.896850Z 2022-01-11T10:01:51.279378Z 2022-01-11T10:02:02.653830Z 2022-01-11T10:01:52.957318Z 2022-01-11T10:01:58.560691Z 2022-01-11T10:01:49.611464Z 2022-01-11T10:01:56.564675Z 2022-01-11T10:01:51.870318Z 2022-01-11T10:01:48.502621Z 2022-01-11T10:01:48.589734Z 2022-01-11T10:01:52.477839Z 2022-01-11T10:01:42.352044Z 2022-01-11T10:01:43.945495Z 2022-01-11T10:01:37.894336Z 2022-01-11T10:01:44.584840Z 2022-01-11T10:01:37.134692Z 2022-01-11T10:01:38.162238Z 2022-01-11T10:07:10.229560Z 2022-01-11T10:07:06.719914Z 2022-01-11T10:07:09.638558Z 2022-01-11T10:07:07.693712Z 2022-01-11T10:07:07.190896Z 2022-01-11T10:07:06.439134Z 2022-01-11T10:07:08.690491Z 2022-01-11T10:07:05.958320Z 2022-01-11T10:07:09.931325Z 2022-01-11T10:07:05.467269Z 2022-01-11T10:07:04.185458Z 2022-01-11T10:07:05.213031Z 2022-01-11T10:07:08.414801Z 2022-01-11T10:07:06.487693Z 2022-01-11T10:07:04.918517Z 2022-01-11T10:07:01.432181Z 2022-01-11T10:07:05.801176Z 2022-01-11T10:07:06.810361Z 2022-01-11T10:07:03.282958Z 2022-01-11T10:07:02.494581Z 2022-01-11T10:07:08.471441Z 2022-01-11T10:07:02.857961Z 2022-01-11T10:06:56.111193Z 2022-01-11T10:07:05.324515Z 2022-01-11T10:07:08.835435Z 2022-01-11T10:07:05.168673Z 2022-01-11T10:07:01.668469Z 2022-01-11T10:07:03.544008Z 2022-01-11T10:07:07.226897Z 2022-01-11T10:07:02.007740Z 2022-01-11T10:07:06.820512Z 2022-01-11T10:07:03.958571Z 2022-01-11T10:06:57.730299Z 2022-01-11T10:07:03.795437Z 2022-01-11T10:07:06.801434Z 2022-01-11T10:07:00.642140Z 2022-01-11T10:07:00.290432Z 2022-01-11T10:06:57.042041Z 2022-01-11T10:07:00.217761Z 2022-01-11T10:06:57.820430Z 2022-01-11T10:07:01.484404Z 2022-01-11T10:07:02.467980Z 2022-01-11T10:07:04.419190Z 2022-01-11T10:07:01.429669Z 2022-01-11T10:06:58.680000Z 2022-01-11T10:06:56.257301Z 2022-01-11T10:06:59.548899Z 2022-01-11T10:07:00.846388Z 2022-01-11T10:06:58.547581Z 2022-01-11T10:06:56.606330Z 2022-01-11T10:06:53.277409Z 2022-01-11T10:06:48.316530Z 2022-01-11T10:06:59.133344Z 2022-01-11T10:06:51.904445Z 2022-01-11T10:07:03.382294Z 2022-01-11T10:06:53.092384Z 2022-01-11T10:06:57.852662Z 2022-01-11T10:06:48.152115Z 2022-01-11T10:07:00.110373Z 2022-01-11T10:06:53.882475Z 2022-01-11T10:06:51.866130Z 2022-01-11T10:06:59.749119Z 2022-01-11T10:06:49.529281Z 2022-01-11T10:06:53.457642Z 2022-01-11T10:06:44.495482Z 2022-01-11T10:06:51.426343Z 2022-01-11T10:06:24.490346Z 2022-01-11T10:06:34.331383Z 2022-01-11T10:06:51.864454Z 2022-01-11T10:06:23.388634Z 2022-01-11T10:06:34.966397Z 2022-01-11T10:06:26.647108Z 2022-01-11T10:06:23.146020Z 2022-01-11T10:06:31.496306Z 2022-01-11T10:06:21.642144Z 2022-01-11T10:06:08.539242Z 2022-01-11T10:06:23.537398Z 2022-01-11T10:06:32.281279Z 2022-01-11T10:06:13.318641Z 2022-01-11T10:06:22.370326Z 2022-01-11T10:06:17.865035Z 2022-01-11T10:06:08.510395Z 2022-01-11T10:06:18.759441Z 2022-01-11T10:06:09.829149Z 2022-01-11T10:06:10.822534Z 2022-01-11T10:06:07.334314Z 2022-01-11T10:06:02.125433Z 2022-01-11T10:06:02.972410Z 2022-01-11T10:06:03.515726Z 2022-01-11T10:06:11.435633Z 2022-01-11T10:06:01.879367Z 2022-01-11T10:05:51.130615Z 2022-01-11T10:05:47.420182Z 2022-01-11T10:05:49.915302Z 2022-01-11T10:05:47.658042Z 2022-01-11T10:05:46.955258Z 2022-01-11T10:05:42.442682Z 2022-01-11T10:05:41.471429Z 2022-01-11T10:05:37.397820Z 2022-01-11T10:05:37.560498Z 2022-01-11T10:11:57.557705Z 2022-01-11T10:12:00.033124Z 2022-01-11T10:11:55.805895Z 2022-01-11T10:11:56.625980Z 2022-01-11T10:11:59.723547Z 2022-01-11T10:11:58.164847Z 2022-01-11T10:12:00.277935Z 2022-01-11T10:11:56.816385Z 2022-01-11T10:11:58.008133Z 2022-01-11T10:11:52.899915Z 2022-01-11T10:11:59.610707Z 2022-01-11T10:11:53.623238Z 2022-01-11T10:11:58.070057Z 2022-01-11T10:11:57.373061Z 2022-01-11T10:11:58.372538Z 2022-01-11T10:11:54.513120Z 2022-01-11T10:11:59.315438Z 2022-01-11T10:11:55.705562Z 2022-01-11T10:11:59.818550Z 2022-01-11T10:11:56.813712Z 2022-01-11T10:11:56.937912Z 2022-01-11T10:11:51.889084Z 2022-01-11T10:11:57.979417Z 2022-01-11T10:11:53.009944Z 2022-01-11T10:11:58.700014Z 2022-01-11T10:11:51.080375Z 2022-01-11T10:11:54.518766Z 2022-01-11T10:11:53.190787Z 2022-01-11T10:11:56.517987Z 2022-01-11T10:11:52.672606Z 2022-01-11T10:11:54.651873Z 2022-01-11T10:11:48.843219Z 2022-01-11T10:11:53.690295Z 2022-01-11T10:11:50.254677Z 2022-01-11T10:11:55.739804Z 2022-01-11T10:11:49.065113Z 2022-01-11T10:11:51.698112Z 2022-01-11T10:11:48.876428Z 2022-01-11T10:11:51.747959Z 2022-01-11T10:11:51.230499Z 2022-01-11T10:11:55.752070Z 2022-01-11T10:11:47.729139Z 2022-01-11T10:11:51.490470Z 2022-01-11T10:11:44.264340Z 2022-01-11T10:11:47.533520Z 2022-01-11T10:11:45.243008Z 2022-01-11T10:11:51.020097Z 2022-01-11T10:11:24.243237Z 2022-01-11T10:11:53.816669Z 2022-01-11T10:11:43.786358Z 2022-01-11T10:11:48.039119Z 2022-01-11T10:11:39.191425Z 2022-01-11T10:11:44.937407Z 2022-01-11T10:11:40.429373Z 2022-01-11T10:11:48.745619Z 2022-01-11T10:11:43.498735Z 2022-01-11T10:11:46.653384Z 2022-01-11T10:11:36.993246Z 2022-01-11T10:11:38.929599Z 2022-01-11T10:11:33.883188Z 2022-01-11T10:11:42.653421Z 2022-01-11T10:11:31.825211Z 2022-01-11T10:11:45.079682Z 2022-01-11T10:11:21.607204Z 2022-01-11T10:11:36.355118Z 2022-01-11T10:11:35.300429Z 2022-01-11T10:11:37.403379Z 2022-01-11T10:11:24.804125Z 2022-01-11T10:11:42.878456Z 2022-01-11T10:11:31.349438Z 2022-01-11T10:10:56.896403Z 2022-01-11T10:11:09.874352Z 2022-01-11T10:10:56.080759Z 2022-01-11T10:11:02.149402Z 2022-01-11T10:10:59.073466Z 2022-01-11T10:11:09.582371Z 2022-01-11T10:10:56.244107Z 2022-01-11T10:11:06.874095Z 2022-01-11T10:10:51.743429Z 2022-01-11T10:11:02.938960Z 2022-01-11T10:10:54.485412Z 2022-01-11T10:11:04.178371Z 2022-01-11T10:10:49.046883Z 2022-01-11T10:10:57.357236Z 2022-01-11T10:10:54.217577Z 2022-01-11T10:10:45.012746Z 2022-01-11T10:10:42.009443Z 2022-01-11T10:10:47.747093Z 2022-01-11T10:10:43.891495Z 2022-01-11T10:10:43.792888Z 2022-01-11T10:10:41.022001Z 2022-01-11T10:10:43.171596Z 2022-01-11T10:10:40.943440Z 2022-01-11T10:10:41.908410Z 2022-01-11T10:10:38.139578Z 2022-01-11T10:10:41.497992Z 2022-01-11T10:10:36.891791Z 2022-01-11T10:10:33.716025Z 2022-01-11T10:10:33.344020Z 2022-01-11T10:10:33.218307Z 2022-01-11T10:16:19.171481Z 2022-01-11T10:16:15.380954Z 2022-01-11T10:16:20.550593Z 2022-01-11T10:16:15.429767Z 2022-01-11T10:16:18.658037Z 2022-01-11T10:16:15.376554Z 2022-01-11T10:16:20.393211Z 2022-01-11T10:16:14.930676Z 2022-01-11T10:16:18.952365Z 2022-01-11T10:16:14.262030Z 2022-01-11T10:16:16.091680Z 2022-01-11T10:16:11.137041Z 2022-01-11T10:16:17.201553Z 2022-01-11T10:16:14.029035Z 2022-01-11T10:16:17.673678Z 2022-01-11T10:16:14.946593Z 2022-01-11T10:16:20.154424Z 2022-01-11T10:16:13.536113Z 2022-01-11T10:16:18.181027Z 2022-01-11T10:16:12.242761Z 2022-01-11T10:16:17.739694Z 2022-01-11T10:16:10.126350Z 2022-01-11T10:16:16.650183Z 2022-01-11T10:16:11.792894Z 2022-01-11T10:16:19.063804Z 2022-01-11T10:16:10.657746Z 2022-01-11T10:16:17.138625Z 2022-01-11T10:16:12.387061Z 2022-01-11T10:16:16.220083Z 2022-01-11T10:16:11.944379Z 2022-01-11T10:16:11.075947Z 2022-01-11T10:16:10.006448Z 2022-01-11T10:16:16.119766Z 2022-01-11T10:16:10.555030Z 2022-01-11T10:16:16.680220Z 2022-01-11T10:16:10.320944Z 2022-01-11T10:16:15.238723Z 2022-01-11T10:16:08.772660Z 2022-01-11T10:16:09.740410Z 2022-01-11T10:16:06.220452Z 2022-01-11T10:16:02.615212Z 2022-01-11T10:16:05.484741Z 2022-01-11T10:16:09.709584Z 2022-01-11T10:15:58.410080Z 2022-01-11T10:16:12.813807Z 2022-01-11T10:16:02.932554Z 2022-01-11T10:16:15.286048Z 2022-01-11T10:16:04.344834Z 2022-01-11T10:16:07.468635Z 2022-01-11T10:16:01.454468Z 2022-01-11T10:16:12.708235Z 2022-01-11T10:16:02.017170Z 2022-01-11T10:16:03.207448Z 2022-01-11T10:15:55.409198Z 2022-01-11T10:16:09.407217Z 2022-01-11T10:15:51.966381Z 2022-01-11T10:16:07.984867Z 2022-01-11T10:15:49.976162Z 2022-01-11T10:15:59.481272Z 2022-01-11T10:15:56.141283Z 2022-01-11T10:15:59.991385Z 2022-01-11T10:15:53.608490Z 2022-01-11T10:16:00.464796Z 2022-01-11T10:15:43.332197Z 2022-01-11T10:15:53.905227Z 2022-01-11T10:15:36.239360Z 2022-01-11T10:15:59.262343Z 2022-01-11T10:15:42.687173Z 2022-01-11T10:15:54.831311Z 2022-01-11T10:15:39.539073Z 2022-01-11T10:15:48.184087Z 2022-01-11T10:15:35.760220Z 2022-01-11T10:15:49.415234Z 2022-01-11T10:15:14.752025Z 2022-01-11T10:15:37.531414Z 2022-01-11T10:15:45.734711Z 2022-01-11T10:15:31.423642Z 2022-01-11T10:15:05.902852Z 2022-01-11T10:15:11.798111Z 2022-01-11T10:15:05.297151Z 2022-01-11T10:15:14.248199Z 2022-01-11T10:15:05.500693Z 2022-01-11T10:15:08.678476Z 2022-01-11T10:15:03.742538Z 2022-01-11T10:15:10.603847Z 2022-01-11T10:15:04.994131Z 2022-01-11T10:15:07.704167Z 2022-01-11T10:15:01.396274Z 2022-01-11T10:15:04.359100Z 2022-01-11T10:15:01.084330Z 2022-01-11T10:14:56.327091Z 2022-01-11T10:14:57.335215Z 2022-01-11T10:14:57.943122Z 2022-01-11T10:14:56.668765Z 2022-01-11T10:14:56.506462Z 2022-01-11T10:14:55.198034Z 2022-01-11T10:14:53.664293Z 2022-01-11T10:14:48.877114Z 2022-01-11T10:14:47.217664Z 2022-01-11T10:14:47.482410Z", + "finished_diff": 102.183038, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 159.105481, + "mean": 51.380770752, + "min": 13.289425 + }, + "jobs_events_duration": { + "max": 6.604843, + "mean": 3.0419838440000024, + "min": 0.164565 + }, + "jobs_events_lag": { + "max": -1.237294, + "mean": -4.881795852000003, + "min": -7.119824 + }, + "jobs_waiting": { + "max": 40.205435, + "mean": 22.84691695000001, + "min": 0.504014 + }, + "modified": "2022-01-11T09:56:59.687123Z 2022-01-11T09:56:59.643742Z 2022-01-11T09:56:59.602557Z 2022-01-11T09:56:59.563859Z 2022-01-11T09:56:59.525590Z 2022-01-11T09:56:59.489424Z 2022-01-11T09:56:59.442214Z 2022-01-11T09:56:59.403017Z 2022-01-11T09:56:59.367964Z 2022-01-11T09:56:59.287727Z 2022-01-11T09:56:59.187320Z 2022-01-11T09:56:59.087716Z 2022-01-11T09:56:59.048505Z 2022-01-11T09:56:59.011057Z 2022-01-11T09:56:58.974656Z 2022-01-11T09:56:53.682643Z 2022-01-11T09:56:53.574734Z 2022-01-11T09:56:53.425134Z 2022-01-11T09:56:53.154518Z 2022-01-11T09:56:52.919719Z 2022-01-11T09:56:52.619284Z 2022-01-11T09:56:52.369367Z 2022-01-11T09:56:52.178068Z 2022-01-11T09:56:51.917529Z 2022-01-11T09:56:49.664924Z 2022-01-11T09:56:49.620827Z 2022-01-11T09:56:49.582695Z 2022-01-11T09:56:49.485446Z 2022-01-11T09:56:49.428007Z 2022-01-11T09:56:48.669505Z 2022-01-11T09:56:48.100830Z 2022-01-11T09:56:47.686227Z 2022-01-11T09:56:47.353300Z 2022-01-11T09:56:46.845628Z 2022-01-11T09:56:46.803547Z 2022-01-11T09:57:41.989452Z 2022-01-11T09:57:41.881502Z 2022-01-11T09:57:41.700805Z 2022-01-11T09:57:41.545554Z 2022-01-11T09:57:24.433870Z 2022-01-11T09:57:24.343573Z 2022-01-11T09:57:24.218835Z 2022-01-11T09:57:24.081834Z 2022-01-11T09:57:23.949571Z 2022-01-11T09:57:23.836470Z 2022-01-11T09:57:23.689006Z 2022-01-11T09:57:23.546868Z 2022-01-11T09:57:23.405459Z 2022-01-11T09:57:23.255597Z 2022-01-11T09:57:23.126113Z 2022-01-11T09:57:23.039339Z 2022-01-11T09:57:22.951357Z 2022-01-11T09:57:22.866398Z 2022-01-11T09:57:22.782357Z 2022-01-11T09:57:22.684313Z 2022-01-11T09:57:22.590653Z 2022-01-11T09:57:22.542361Z 2022-01-11T09:57:22.485791Z 2022-01-11T09:57:22.429644Z 2022-01-11T09:57:22.354827Z 2022-01-11T09:57:22.284106Z 2022-01-11T09:57:22.207585Z 2022-01-11T09:57:00.952954Z 2022-01-11T09:57:00.919264Z 2022-01-11T09:57:00.885949Z 2022-01-11T09:57:00.852273Z 2022-01-11T09:57:00.805753Z 2022-01-11T09:57:00.752102Z 2022-01-11T09:57:00.691380Z 2022-01-11T09:57:21.991857Z 2022-01-11T09:57:00.656739Z 2022-01-11T09:57:21.767959Z 2022-01-11T09:57:00.624886Z 2022-01-11T09:57:00.590512Z 2022-01-11T09:57:00.556028Z 2022-01-11T09:57:00.523080Z 2022-01-11T09:57:21.281364Z 2022-01-11T09:57:00.488249Z 2022-01-11T09:57:00.453761Z 2022-01-11T09:57:00.420632Z 2022-01-11T09:57:00.388479Z 2022-01-11T09:57:00.354546Z 2022-01-11T09:57:00.320950Z 2022-01-11T09:57:00.276848Z 2022-01-11T09:57:00.245246Z 2022-01-11T09:57:00.212900Z 2022-01-11T09:57:00.175920Z 2022-01-11T09:57:00.145207Z 2022-01-11T09:57:00.115228Z 2022-01-11T09:57:00.081678Z 2022-01-11T09:57:00.051444Z 2022-01-11T09:57:00.020999Z 2022-01-11T09:56:59.991248Z 2022-01-11T09:56:59.959230Z 2022-01-11T09:56:59.926916Z 2022-01-11T09:56:59.896010Z 2022-01-11T09:56:59.859903Z 2022-01-11T09:56:59.825300Z 2022-01-11T09:56:59.778843Z 2022-01-11T09:56:59.724231Z 2022-01-11T10:02:04.237458Z 2022-01-11T10:02:04.079610Z 2022-01-11T10:02:03.905579Z 2022-01-11T10:02:03.722460Z 2022-01-11T10:02:03.569497Z 2022-01-11T10:02:03.438003Z 2022-01-11T10:02:03.333383Z 2022-01-11T10:02:03.212447Z 2022-01-11T10:02:03.104409Z 2022-01-11T10:02:02.984874Z 2022-01-11T10:02:02.837506Z 2022-01-11T10:02:02.696885Z 2022-01-11T10:02:02.589225Z 2022-01-11T10:02:02.498834Z 2022-01-11T10:02:02.372269Z 2022-01-11T10:02:02.267497Z 2022-01-11T10:02:02.197420Z 2022-01-11T10:02:02.126401Z 2022-01-11T10:02:02.032407Z 2022-01-11T10:02:01.940467Z 2022-01-11T10:02:01.828519Z 2022-01-11T10:02:01.710383Z 2022-01-11T10:02:01.595565Z 2022-01-11T10:02:01.504499Z 2022-01-11T10:02:01.403428Z 2022-01-11T10:02:01.302426Z 2022-01-11T10:02:01.207492Z 2022-01-11T10:02:01.118498Z 2022-01-11T10:02:01.012553Z 2022-01-11T10:02:00.839353Z 2022-01-11T10:02:00.701522Z 2022-01-11T10:02:00.567637Z 2022-01-11T10:02:00.421476Z 2022-01-11T10:02:00.297494Z 2022-01-11T10:01:41.010547Z 2022-01-11T10:02:00.125763Z 2022-01-11T10:01:40.971866Z 2022-01-11T10:01:40.932703Z 2022-01-11T10:01:40.899514Z 2022-01-11T10:01:40.867954Z 2022-01-11T10:01:40.831029Z 2022-01-11T10:01:40.764449Z 2022-01-11T10:01:40.667782Z 2022-01-11T10:01:40.556691Z 2022-01-11T10:01:40.455505Z 2022-01-11T10:01:40.382526Z 2022-01-11T10:01:40.320426Z 2022-01-11T10:01:40.245587Z 2022-01-11T10:01:40.165492Z 2022-01-11T10:01:40.055361Z 2022-01-11T10:01:39.968707Z 2022-01-11T10:01:39.913318Z 2022-01-11T10:01:39.874515Z 2022-01-11T10:01:39.838417Z 2022-01-11T10:01:39.798501Z 2022-01-11T10:01:39.733812Z 2022-01-11T10:01:39.687626Z 2022-01-11T10:01:39.628353Z 2022-01-11T10:01:39.535814Z 2022-01-11T10:01:39.414560Z 2022-01-11T10:01:39.349558Z 2022-01-11T10:01:39.300385Z 2022-01-11T10:01:39.226262Z 2022-01-11T10:01:39.163641Z 2022-01-11T10:01:39.090355Z 2022-01-11T10:01:38.958602Z 2022-01-11T10:01:38.843594Z 2022-01-11T10:01:38.743656Z 2022-01-11T10:01:38.641884Z 2022-01-11T10:01:38.564157Z 2022-01-11T10:01:38.350467Z 2022-01-11T10:01:38.171538Z 2022-01-11T10:01:38.052494Z 2022-01-11T10:01:37.882478Z 2022-01-11T10:01:37.734401Z 2022-01-11T10:01:37.622509Z 2022-01-11T10:01:37.427482Z 2022-01-11T10:01:37.231514Z 2022-01-11T10:01:37.035561Z 2022-01-11T10:01:25.831603Z 2022-01-11T10:01:36.859505Z 2022-01-11T10:01:36.619412Z 2022-01-11T10:01:36.418646Z 2022-01-11T10:01:25.642889Z 2022-01-11T10:01:25.459602Z 2022-01-11T10:01:25.358358Z 2022-01-11T10:01:25.226990Z 2022-01-11T10:01:25.126198Z 2022-01-11T10:01:24.871270Z 2022-01-11T10:01:24.590336Z 2022-01-11T10:01:24.398722Z 2022-01-11T10:01:24.221110Z 2022-01-11T10:01:24.021907Z 2022-01-11T10:01:23.790181Z 2022-01-11T10:01:21.910306Z 2022-01-11T10:01:21.859306Z 2022-01-11T10:01:21.798414Z 2022-01-11T10:01:21.735596Z 2022-01-11T10:01:21.677386Z 2022-01-11T10:01:21.599591Z 2022-01-11T10:05:50.510284Z 2022-01-11T10:05:50.458482Z 2022-01-11T10:05:50.392679Z 2022-01-11T10:05:50.322391Z 2022-01-11T10:05:50.251505Z 2022-01-11T10:05:50.183384Z 2022-01-11T10:05:50.074331Z 2022-01-11T10:05:49.991279Z 2022-01-11T10:05:49.913233Z 2022-01-11T10:05:49.825423Z 2022-01-11T10:05:49.750313Z 2022-01-11T10:05:49.666289Z 2022-01-11T10:05:49.578319Z 2022-01-11T10:05:49.508225Z 2022-01-11T10:05:49.464205Z 2022-01-11T10:05:49.422188Z 2022-01-11T10:05:49.377144Z 2022-01-11T10:05:49.333227Z 2022-01-11T10:05:49.272270Z 2022-01-11T10:05:49.195267Z 2022-01-11T10:05:49.118316Z 2022-01-11T10:05:49.040499Z 2022-01-11T10:05:48.989206Z 2022-01-11T10:05:48.934294Z 2022-01-11T10:05:48.883521Z 2022-01-11T10:05:48.827236Z 2022-01-11T10:05:48.772488Z 2022-01-11T10:05:48.719365Z 2022-01-11T10:05:48.669350Z 2022-01-11T10:05:48.617350Z 2022-01-11T10:05:48.567292Z 2022-01-11T10:05:48.513287Z 2022-01-11T10:05:48.465397Z 2022-01-11T10:05:48.408454Z 2022-01-11T10:05:48.338630Z 2022-01-11T10:05:48.276235Z 2022-01-11T10:05:48.198435Z 2022-01-11T10:05:48.089725Z 2022-01-11T10:05:47.978510Z 2022-01-11T10:05:47.880731Z 2022-01-11T10:05:47.786303Z 2022-01-11T10:05:47.683256Z 2022-01-11T10:05:47.584335Z 2022-01-11T10:05:47.478371Z 2022-01-11T10:05:47.384274Z 2022-01-11T10:05:47.296334Z 2022-01-11T10:05:47.209456Z 2022-01-11T10:05:47.117337Z 2022-01-11T10:05:47.014368Z 2022-01-11T10:05:46.929329Z 2022-01-11T10:05:46.846342Z 2022-01-11T10:05:46.758499Z 2022-01-11T10:05:46.659611Z 2022-01-11T10:05:46.575541Z 2022-01-11T10:05:46.510020Z 2022-01-11T10:05:46.458432Z 2022-01-11T10:05:46.385456Z 2022-01-11T10:05:46.336351Z 2022-01-11T10:05:46.285398Z 2022-01-11T10:05:46.228720Z 2022-01-11T10:05:46.112488Z 2022-01-11T10:05:46.170387Z 2022-01-11T10:05:46.042403Z 2022-01-11T10:05:45.975399Z 2022-01-11T10:05:45.866742Z 2022-01-11T10:05:45.915204Z 2022-01-11T10:05:33.176338Z 2022-01-11T10:05:32.886434Z 2022-01-11T10:05:45.827492Z 2022-01-11T10:05:32.653882Z 2022-01-11T10:05:32.330436Z 2022-01-11T10:05:32.093329Z 2022-01-11T10:05:31.764066Z 2022-01-11T10:05:31.513918Z 2022-01-11T10:05:31.186964Z 2022-01-11T10:05:30.931908Z 2022-01-11T10:05:30.681436Z 2022-01-11T10:05:30.433260Z 2022-01-11T10:05:30.289886Z 2022-01-11T10:05:30.034210Z 2022-01-11T10:05:29.883735Z 2022-01-11T10:05:29.738502Z 2022-01-11T10:05:29.607321Z 2022-01-11T10:05:29.518530Z 2022-01-11T10:05:29.425243Z 2022-01-11T10:05:29.330056Z 2022-01-11T10:05:29.232411Z 2022-01-11T10:05:29.122390Z 2022-01-11T10:05:29.003308Z 2022-01-11T10:05:28.914295Z 2022-01-11T10:05:28.826271Z 2022-01-11T10:05:25.919919Z 2022-01-11T10:05:25.800661Z 2022-01-11T10:05:25.653036Z 2022-01-11T10:05:25.530769Z 2022-01-11T10:05:25.394834Z 2022-01-11T10:05:24.472080Z 2022-01-11T10:05:24.404946Z 2022-01-11T10:05:23.784744Z 2022-01-11T10:05:23.731130Z 2022-01-11T10:11:10.022617Z 2022-01-11T10:10:42.291808Z 2022-01-11T10:10:42.246711Z 2022-01-11T10:11:09.885344Z 2022-01-11T10:10:42.205320Z 2022-01-11T10:10:42.168487Z 2022-01-11T10:10:42.134150Z 2022-01-11T10:10:42.098526Z 2022-01-11T10:10:42.063182Z 2022-01-11T10:10:42.029994Z 2022-01-11T10:10:41.992616Z 2022-01-11T10:10:41.906435Z 2022-01-11T10:10:41.732437Z 2022-01-11T10:10:41.665440Z 2022-01-11T10:10:41.594519Z 2022-01-11T10:10:41.537310Z 2022-01-11T10:10:41.489836Z 2022-01-11T10:10:41.439465Z 2022-01-11T10:10:41.383294Z 2022-01-11T10:10:41.323912Z 2022-01-11T10:10:41.274745Z 2022-01-11T10:10:41.217167Z 2022-01-11T10:10:41.153330Z 2022-01-11T10:10:41.068498Z 2022-01-11T10:10:40.966940Z 2022-01-11T10:10:40.847618Z 2022-01-11T10:10:40.726621Z 2022-01-11T10:10:40.632419Z 2022-01-11T10:10:40.499577Z 2022-01-11T10:10:40.369847Z 2022-01-11T10:10:40.242616Z 2022-01-11T10:10:40.146988Z 2022-01-11T10:10:40.075651Z 2022-01-11T10:10:39.991436Z 2022-01-11T10:10:39.885283Z 2022-01-11T10:10:39.788739Z 2022-01-11T10:10:39.673867Z 2022-01-11T10:10:39.561533Z 2022-01-11T10:10:39.457177Z 2022-01-11T10:10:39.332529Z 2022-01-11T10:10:39.211513Z 2022-01-11T10:10:39.091773Z 2022-01-11T10:10:39.009409Z 2022-01-11T10:10:38.859366Z 2022-01-11T10:10:38.724907Z 2022-01-11T10:10:38.627769Z 2022-01-11T10:10:38.538619Z 2022-01-11T10:10:38.420509Z 2022-01-11T10:10:38.332049Z 2022-01-11T10:10:38.290921Z 2022-01-11T10:10:38.210624Z 2022-01-11T10:10:38.162678Z 2022-01-11T10:10:38.115019Z 2022-01-11T10:10:38.056384Z 2022-01-11T10:10:37.950124Z 2022-01-11T10:10:37.904521Z 2022-01-11T10:10:37.857333Z 2022-01-11T10:10:37.811751Z 2022-01-11T10:10:37.742615Z 2022-01-11T10:10:37.676456Z 2022-01-11T10:10:37.610460Z 2022-01-11T10:10:37.537490Z 2022-01-11T10:10:37.462072Z 2022-01-11T10:10:37.387550Z 2022-01-11T10:10:37.305643Z 2022-01-11T10:10:37.203568Z 2022-01-11T10:10:37.111723Z 2022-01-11T10:10:36.969576Z 2022-01-11T10:10:36.788620Z 2022-01-11T10:10:36.658720Z 2022-01-11T10:10:24.065016Z 2022-01-11T10:10:23.948410Z 2022-01-11T10:10:23.830044Z 2022-01-11T10:10:23.567461Z 2022-01-11T10:10:23.352517Z 2022-01-11T10:10:23.158594Z 2022-01-11T10:10:22.985214Z 2022-01-11T10:10:22.824038Z 2022-01-11T10:10:22.529441Z 2022-01-11T10:10:22.304730Z 2022-01-11T10:10:22.110397Z 2022-01-11T10:10:21.925128Z 2022-01-11T10:10:21.753373Z 2022-01-11T10:10:21.493040Z 2022-01-11T10:10:21.222608Z 2022-01-11T10:10:16.382030Z 2022-01-11T10:10:16.300621Z 2022-01-11T10:10:16.240392Z 2022-01-11T10:10:16.158905Z 2022-01-11T10:10:16.066473Z 2022-01-11T10:10:15.949687Z 2022-01-11T10:10:15.881531Z 2022-01-11T10:10:15.809348Z 2022-01-11T10:10:15.711057Z 2022-01-11T10:10:15.642412Z 2022-01-11T10:10:15.561918Z 2022-01-11T10:10:14.295770Z 2022-01-11T10:10:14.260582Z 2022-01-11T10:10:14.215287Z 2022-01-11T10:10:14.185846Z 2022-01-11T10:14:56.036870Z 2022-01-11T10:14:55.970510Z 2022-01-11T10:14:55.886486Z 2022-01-11T10:14:55.766291Z 2022-01-11T10:14:55.697306Z 2022-01-11T10:14:55.647329Z 2022-01-11T10:14:55.584286Z 2022-01-11T10:14:55.514208Z 2022-01-11T10:14:55.447277Z 2022-01-11T10:14:55.403298Z 2022-01-11T10:14:55.365385Z 2022-01-11T10:14:55.329108Z 2022-01-11T10:14:55.290215Z 2022-01-11T10:14:55.252358Z 2022-01-11T10:14:55.200885Z 2022-01-11T10:14:55.167404Z 2022-01-11T10:14:55.124531Z 2022-01-11T10:14:55.076264Z 2022-01-11T10:14:55.017192Z 2022-01-11T10:14:54.953222Z 2022-01-11T10:14:54.889381Z 2022-01-11T10:14:54.824295Z 2022-01-11T10:14:54.755553Z 2022-01-11T10:14:54.684150Z 2022-01-11T10:14:54.622279Z 2022-01-11T10:14:54.547278Z 2022-01-11T10:14:54.455582Z 2022-01-11T10:14:54.380499Z 2022-01-11T10:14:54.283424Z 2022-01-11T10:14:54.212506Z 2022-01-11T10:14:54.125377Z 2022-01-11T10:14:54.071512Z 2022-01-11T10:14:54.001479Z 2022-01-11T10:14:53.945410Z 2022-01-11T10:14:53.867550Z 2022-01-11T10:14:53.801276Z 2022-01-11T10:14:53.738252Z 2022-01-11T10:14:53.671052Z 2022-01-11T10:14:53.593531Z 2022-01-11T10:14:53.543883Z 2022-01-11T10:14:53.494287Z 2022-01-11T10:14:53.453759Z 2022-01-11T10:14:53.416361Z 2022-01-11T10:14:53.302486Z 2022-01-11T10:14:53.237674Z 2022-01-11T10:14:53.172355Z 2022-01-11T10:14:53.111275Z 2022-01-11T10:14:53.024336Z 2022-01-11T10:14:52.977191Z 2022-01-11T10:14:52.869302Z 2022-01-11T10:14:52.802507Z 2022-01-11T10:14:52.726502Z 2022-01-11T10:14:52.642766Z 2022-01-11T10:14:52.566551Z 2022-01-11T10:14:52.501665Z 2022-01-11T10:14:52.419597Z 2022-01-11T10:14:52.375448Z 2022-01-11T10:14:52.320816Z 2022-01-11T10:14:52.248317Z 2022-01-11T10:14:52.175978Z 2022-01-11T10:14:52.045918Z 2022-01-11T10:14:51.941712Z 2022-01-11T10:14:51.815648Z 2022-01-11T10:14:51.671566Z 2022-01-11T10:14:51.539482Z 2022-01-11T10:14:51.410976Z 2022-01-11T10:14:51.305486Z 2022-01-11T10:14:51.151480Z 2022-01-11T10:14:51.042482Z 2022-01-11T10:14:50.924476Z 2022-01-11T10:14:50.826791Z 2022-01-11T10:14:50.715618Z 2022-01-11T10:14:50.620116Z 2022-01-11T10:14:40.108539Z 2022-01-11T10:14:50.518251Z 2022-01-11T10:14:50.403408Z 2022-01-11T10:14:50.189810Z 2022-01-11T10:14:40.000688Z 2022-01-11T10:14:39.906919Z 2022-01-11T10:14:39.750974Z 2022-01-11T10:14:39.281593Z 2022-01-11T10:14:38.857797Z 2022-01-11T10:14:38.443366Z 2022-01-11T10:14:38.273155Z 2022-01-11T10:14:37.972427Z 2022-01-11T10:14:37.649610Z 2022-01-11T10:14:37.399199Z 2022-01-11T10:14:37.263310Z 2022-01-11T10:14:36.972860Z 2022-01-11T10:14:36.581733Z 2022-01-11T10:14:33.023616Z 2022-01-11T10:14:32.955418Z 2022-01-11T10:14:32.868776Z 2022-01-11T10:14:32.805429Z 2022-01-11T10:14:32.736608Z 2022-01-11T10:14:32.629858Z 2022-01-11T10:14:31.810471Z 2022-01-11T10:14:31.774078Z 2022-01-11T10:14:31.730642Z 2022-01-11T10:14:31.693017Z", + "modified_diff": 40.95671, + "started": "2022-01-11T09:57:05.407619Z 2022-01-11T09:57:03.282144Z 2022-01-11T09:57:04.602039Z 2022-01-11T09:57:02.780687Z 2022-01-11T09:57:04.793159Z 2022-01-11T09:57:02.473490Z 2022-01-11T09:57:03.272709Z 2022-01-11T09:57:02.143968Z 2022-01-11T09:57:03.646275Z 2022-01-11T09:57:01.763613Z 2022-01-11T09:57:02.648252Z 2022-01-11T09:57:01.627310Z 2022-01-11T09:57:01.960910Z 2022-01-11T09:57:01.331326Z 2022-01-11T09:57:01.729078Z 2022-01-11T09:56:57.978829Z 2022-01-11T09:56:58.202566Z 2022-01-11T09:56:57.083367Z 2022-01-11T09:56:57.188805Z 2022-01-11T09:56:56.667676Z 2022-01-11T09:56:56.441170Z 2022-01-11T09:56:55.807545Z 2022-01-11T09:56:55.475894Z 2022-01-11T09:56:54.765933Z 2022-01-11T09:56:51.014413Z 2022-01-11T09:56:50.527329Z 2022-01-11T09:56:50.638162Z 2022-01-11T09:56:50.123385Z 2022-01-11T09:56:50.103747Z 2022-01-11T09:56:48.928522Z 2022-01-11T09:56:48.397175Z 2022-01-11T09:56:47.922357Z 2022-01-11T09:56:47.536279Z 2022-01-11T09:56:47.085721Z 2022-01-11T09:56:47.011701Z 2022-01-11T09:57:43.352146Z 2022-01-11T09:57:43.019381Z 2022-01-11T09:57:42.714816Z 2022-01-11T09:57:42.474159Z 2022-01-11T09:57:33.002447Z 2022-01-11T09:57:32.459537Z 2022-01-11T09:57:32.268402Z 2022-01-11T09:57:31.741094Z 2022-01-11T09:57:31.634088Z 2022-01-11T09:57:30.962103Z 2022-01-11T09:57:30.460695Z 2022-01-11T09:57:30.288322Z 2022-01-11T09:57:30.675664Z 2022-01-11T09:57:29.804695Z 2022-01-11T09:57:29.285110Z 2022-01-11T09:57:29.148097Z 2022-01-11T09:57:28.961249Z 2022-01-11T09:57:28.582721Z 2022-01-11T09:57:28.741181Z 2022-01-11T09:57:27.999083Z 2022-01-11T09:57:28.099654Z 2022-01-11T09:57:27.412083Z 2022-01-11T09:57:27.218332Z 2022-01-11T09:57:26.689048Z 2022-01-11T09:57:26.607725Z 2022-01-11T09:57:26.065330Z 2022-01-11T09:57:25.929257Z 2022-01-11T09:57:14.286502Z 2022-01-11T09:57:14.924874Z 2022-01-11T09:57:13.766237Z 2022-01-11T09:57:14.097389Z 2022-01-11T09:57:13.226668Z 2022-01-11T09:57:13.626611Z 2022-01-11T09:57:12.548011Z 2022-01-11T09:57:25.714687Z 2022-01-11T09:57:13.002604Z 2022-01-11T09:57:25.291657Z 2022-01-11T09:57:11.774742Z 2022-01-11T09:57:12.476356Z 2022-01-11T09:57:10.971196Z 2022-01-11T09:57:12.035196Z 2022-01-11T09:57:25.059367Z 2022-01-11T09:57:10.357765Z 2022-01-11T09:57:12.041673Z 2022-01-11T09:57:09.632212Z 2022-01-11T09:57:10.669801Z 2022-01-11T09:57:09.095851Z 2022-01-11T09:57:10.029915Z 2022-01-11T09:57:08.405263Z 2022-01-11T09:57:10.044336Z 2022-01-11T09:57:07.676637Z 2022-01-11T09:57:09.803986Z 2022-01-11T09:57:07.345420Z 2022-01-11T09:57:08.015934Z 2022-01-11T09:57:06.858722Z 2022-01-11T09:57:06.945650Z 2022-01-11T09:57:06.472137Z 2022-01-11T09:57:07.034130Z 2022-01-11T09:57:05.775628Z 2022-01-11T09:57:07.090291Z 2022-01-11T09:57:04.778565Z 2022-01-11T09:57:06.062541Z 2022-01-11T09:57:04.248650Z 2022-01-11T09:57:05.405698Z 2022-01-11T09:57:03.430121Z 2022-01-11T10:02:14.973272Z 2022-01-11T10:02:14.437470Z 2022-01-11T10:02:14.299381Z 2022-01-11T10:02:13.862352Z 2022-01-11T10:02:13.667077Z 2022-01-11T10:02:13.318359Z 2022-01-11T10:02:12.945299Z 2022-01-11T10:02:12.658840Z 2022-01-11T10:02:12.523965Z 2022-01-11T10:02:11.960327Z 2022-01-11T10:02:11.900693Z 2022-01-11T10:02:11.464020Z 2022-01-11T10:02:11.038382Z 2022-01-11T10:02:10.783306Z 2022-01-11T10:02:10.483265Z 2022-01-11T10:02:09.931288Z 2022-01-11T10:02:09.748734Z 2022-01-11T10:02:09.509844Z 2022-01-11T10:02:09.472283Z 2022-01-11T10:02:08.894477Z 2022-01-11T10:02:08.770461Z 2022-01-11T10:02:08.262129Z 2022-01-11T10:02:08.068748Z 2022-01-11T10:02:07.800166Z 2022-01-11T10:02:07.747265Z 2022-01-11T10:02:07.315417Z 2022-01-11T10:02:06.760234Z 2022-01-11T10:02:06.579903Z 2022-01-11T10:02:06.344257Z 2022-01-11T10:02:05.934140Z 2022-01-11T10:02:05.716413Z 2022-01-11T10:02:05.396999Z 2022-01-11T10:02:05.366323Z 2022-01-11T10:02:05.057437Z 2022-01-11T10:01:58.460269Z 2022-01-11T10:02:04.826280Z 2022-01-11T10:01:57.810222Z 2022-01-11T10:01:57.277296Z 2022-01-11T10:01:57.003225Z 2022-01-11T10:01:57.072426Z 2022-01-11T10:01:56.334039Z 2022-01-11T10:01:55.908813Z 2022-01-11T10:01:55.204261Z 2022-01-11T10:01:54.700082Z 2022-01-11T10:01:54.434471Z 2022-01-11T10:01:54.454650Z 2022-01-11T10:01:53.867190Z 2022-01-11T10:01:53.325879Z 2022-01-11T10:01:52.838255Z 2022-01-11T10:01:52.701603Z 2022-01-11T10:01:52.280103Z 2022-01-11T10:01:52.158684Z 2022-01-11T10:01:51.538320Z 2022-01-11T10:01:50.935305Z 2022-01-11T10:01:50.426519Z 2022-01-11T10:01:49.883762Z 2022-01-11T10:01:49.418499Z 2022-01-11T10:01:49.113755Z 2022-01-11T10:01:48.664962Z 2022-01-11T10:01:48.255285Z 2022-01-11T10:01:47.963142Z 2022-01-11T10:01:47.500655Z 2022-01-11T10:01:47.128962Z 2022-01-11T10:01:46.438327Z 2022-01-11T10:01:46.241327Z 2022-01-11T10:01:46.070103Z 2022-01-11T10:01:45.958682Z 2022-01-11T10:01:45.547954Z 2022-01-11T10:01:45.162421Z 2022-01-11T10:01:44.684845Z 2022-01-11T10:01:44.495293Z 2022-01-11T10:01:44.327549Z 2022-01-11T10:01:43.802304Z 2022-01-11T10:01:43.676355Z 2022-01-11T10:01:43.327936Z 2022-01-11T10:01:42.947283Z 2022-01-11T10:01:42.990035Z 2022-01-11T10:01:42.470037Z 2022-01-11T10:01:42.181009Z 2022-01-11T10:01:30.709495Z 2022-01-11T10:01:42.032357Z 2022-01-11T10:01:41.776331Z 2022-01-11T10:01:41.582000Z 2022-01-11T10:01:30.349866Z 2022-01-11T10:01:29.469308Z 2022-01-11T10:01:29.061503Z 2022-01-11T10:01:28.048081Z 2022-01-11T10:01:28.528378Z 2022-01-11T10:01:27.482827Z 2022-01-11T10:01:27.002911Z 2022-01-11T10:01:27.053954Z 2022-01-11T10:01:26.541396Z 2022-01-11T10:01:26.308463Z 2022-01-11T10:01:26.517703Z 2022-01-11T10:01:23.175813Z 2022-01-11T10:01:23.288168Z 2022-01-11T10:01:22.938510Z 2022-01-11T10:01:23.389878Z 2022-01-11T10:01:22.356201Z 2022-01-11T10:01:22.730242Z 2022-01-11T10:06:16.250828Z 2022-01-11T10:06:15.643320Z 2022-01-11T10:06:15.273380Z 2022-01-11T10:06:14.625269Z 2022-01-11T10:06:14.054069Z 2022-01-11T10:06:13.357234Z 2022-01-11T10:06:13.447551Z 2022-01-11T10:06:12.729987Z 2022-01-11T10:06:12.193693Z 2022-01-11T10:06:11.407475Z 2022-01-11T10:06:11.563191Z 2022-01-11T10:06:10.623617Z 2022-01-11T10:06:10.270438Z 2022-01-11T10:06:09.726058Z 2022-01-11T10:06:09.582600Z 2022-01-11T10:06:09.007977Z 2022-01-11T10:06:08.803219Z 2022-01-11T10:06:08.326235Z 2022-01-11T10:06:08.271204Z 2022-01-11T10:06:07.678691Z 2022-01-11T10:06:07.861716Z 2022-01-11T10:06:07.364284Z 2022-01-11T10:06:07.297793Z 2022-01-11T10:06:06.225155Z 2022-01-11T10:06:05.492146Z 2022-01-11T10:06:05.083423Z 2022-01-11T10:06:04.675935Z 2022-01-11T10:06:03.901281Z 2022-01-11T10:06:03.594048Z 2022-01-11T10:06:02.839030Z 2022-01-11T10:06:02.248996Z 2022-01-11T10:06:01.731515Z 2022-01-11T10:06:01.357303Z 2022-01-11T10:06:01.139550Z 2022-01-11T10:06:00.811395Z 2022-01-11T10:06:00.702587Z 2022-01-11T10:06:00.405231Z 2022-01-11T10:06:00.003605Z 2022-01-11T10:06:00.027465Z 2022-01-11T10:05:59.323158Z 2022-01-11T10:05:59.355971Z 2022-01-11T10:05:58.974974Z 2022-01-11T10:05:58.792385Z 2022-01-11T10:05:58.275100Z 2022-01-11T10:05:58.277443Z 2022-01-11T10:05:57.633270Z 2022-01-11T10:05:57.486915Z 2022-01-11T10:05:56.673846Z 2022-01-11T10:05:56.617342Z 2022-01-11T10:05:55.919494Z 2022-01-11T10:05:55.523702Z 2022-01-11T10:05:54.835189Z 2022-01-11T10:05:54.827397Z 2022-01-11T10:05:54.444995Z 2022-01-11T10:05:54.028735Z 2022-01-11T10:05:53.737330Z 2022-01-11T10:05:53.565057Z 2022-01-11T10:05:53.073290Z 2022-01-11T10:05:52.822873Z 2022-01-11T10:05:52.386694Z 2022-01-11T10:05:51.890085Z 2022-01-11T10:05:52.098282Z 2022-01-11T10:05:51.707671Z 2022-01-11T10:05:51.348154Z 2022-01-11T10:05:50.949997Z 2022-01-11T10:05:51.250190Z 2022-01-11T10:05:42.963726Z 2022-01-11T10:05:42.452307Z 2022-01-11T10:05:50.866230Z 2022-01-11T10:05:42.347426Z 2022-01-11T10:05:41.802856Z 2022-01-11T10:05:41.194170Z 2022-01-11T10:05:41.063907Z 2022-01-11T10:05:40.655844Z 2022-01-11T10:05:40.240754Z 2022-01-11T10:05:39.573272Z 2022-01-11T10:05:39.437440Z 2022-01-11T10:05:39.142069Z 2022-01-11T10:05:38.946313Z 2022-01-11T10:05:38.582658Z 2022-01-11T10:05:38.467483Z 2022-01-11T10:05:37.905131Z 2022-01-11T10:05:37.991526Z 2022-01-11T10:05:37.606745Z 2022-01-11T10:05:37.372108Z 2022-01-11T10:05:36.836531Z 2022-01-11T10:05:36.360329Z 2022-01-11T10:05:35.692072Z 2022-01-11T10:05:35.585071Z 2022-01-11T10:05:35.078416Z 2022-01-11T10:05:34.474551Z 2022-01-11T10:05:28.083193Z 2022-01-11T10:05:28.120346Z 2022-01-11T10:05:27.445336Z 2022-01-11T10:05:26.909882Z 2022-01-11T10:05:26.595256Z 2022-01-11T10:05:25.128561Z 2022-01-11T10:05:24.846802Z 2022-01-11T10:05:24.108395Z 2022-01-11T10:05:24.010144Z 2022-01-11T10:11:11.076985Z 2022-01-11T10:11:08.218953Z 2022-01-11T10:11:08.320759Z 2022-01-11T10:11:10.675979Z 2022-01-11T10:11:07.780645Z 2022-01-11T10:11:07.351743Z 2022-01-11T10:11:06.744406Z 2022-01-11T10:11:06.505127Z 2022-01-11T10:11:05.999695Z 2022-01-11T10:11:05.605440Z 2022-01-11T10:11:05.109121Z 2022-01-11T10:11:04.799480Z 2022-01-11T10:11:04.744940Z 2022-01-11T10:11:03.787876Z 2022-01-11T10:11:03.279184Z 2022-01-11T10:11:02.834656Z 2022-01-11T10:11:02.641786Z 2022-01-11T10:11:01.904319Z 2022-01-11T10:11:01.958413Z 2022-01-11T10:11:01.127460Z 2022-01-11T10:11:00.593335Z 2022-01-11T10:11:00.212943Z 2022-01-11T10:10:59.992987Z 2022-01-11T10:10:59.244088Z 2022-01-11T10:10:58.965609Z 2022-01-11T10:10:58.305127Z 2022-01-11T10:10:58.123427Z 2022-01-11T10:10:57.776461Z 2022-01-11T10:10:57.792972Z 2022-01-11T10:10:57.143685Z 2022-01-11T10:10:56.581257Z 2022-01-11T10:10:55.955332Z 2022-01-11T10:10:56.330856Z 2022-01-11T10:10:55.570242Z 2022-01-11T10:10:55.615097Z 2022-01-11T10:10:54.779792Z 2022-01-11T10:10:54.722471Z 2022-01-11T10:10:53.825323Z 2022-01-11T10:10:53.486481Z 2022-01-11T10:10:53.158927Z 2022-01-11T10:10:53.080130Z 2022-01-11T10:10:52.771544Z 2022-01-11T10:10:52.413647Z 2022-01-11T10:10:52.087200Z 2022-01-11T10:10:51.577037Z 2022-01-11T10:10:51.285187Z 2022-01-11T10:10:50.687911Z 2022-01-11T10:10:50.153743Z 2022-01-11T10:10:49.977324Z 2022-01-11T10:10:49.554353Z 2022-01-11T10:10:49.092445Z 2022-01-11T10:10:48.404900Z 2022-01-11T10:10:48.236304Z 2022-01-11T10:10:47.761027Z 2022-01-11T10:10:47.589260Z 2022-01-11T10:10:47.234768Z 2022-01-11T10:10:46.574249Z 2022-01-11T10:10:46.344197Z 2022-01-11T10:10:45.896412Z 2022-01-11T10:10:45.748098Z 2022-01-11T10:10:45.399891Z 2022-01-11T10:10:45.125694Z 2022-01-11T10:10:44.686887Z 2022-01-11T10:10:44.341898Z 2022-01-11T10:10:44.259254Z 2022-01-11T10:10:43.599115Z 2022-01-11T10:10:43.469015Z 2022-01-11T10:10:43.209010Z 2022-01-11T10:10:43.129504Z 2022-01-11T10:10:42.856920Z 2022-01-11T10:10:34.431251Z 2022-01-11T10:10:33.371141Z 2022-01-11T10:10:33.077123Z 2022-01-11T10:10:32.718173Z 2022-01-11T10:10:32.572189Z 2022-01-11T10:10:32.141329Z 2022-01-11T10:10:32.025325Z 2022-01-11T10:10:31.516637Z 2022-01-11T10:10:30.683811Z 2022-01-11T10:10:30.710061Z 2022-01-11T10:10:29.973257Z 2022-01-11T10:10:29.483894Z 2022-01-11T10:10:28.597917Z 2022-01-11T10:10:28.408334Z 2022-01-11T10:10:26.801843Z 2022-01-11T10:10:18.602932Z 2022-01-11T10:10:18.368031Z 2022-01-11T10:10:18.048325Z 2022-01-11T10:10:17.821067Z 2022-01-11T10:10:17.466097Z 2022-01-11T10:10:17.315942Z 2022-01-11T10:10:17.266002Z 2022-01-11T10:10:17.091503Z 2022-01-11T10:10:16.941871Z 2022-01-11T10:10:16.801894Z 2022-01-11T10:10:16.669684Z 2022-01-11T10:10:15.178781Z 2022-01-11T10:10:15.044432Z 2022-01-11T10:10:14.807566Z 2022-01-11T10:10:14.558459Z 2022-01-11T10:15:24.531245Z 2022-01-11T10:15:24.147623Z 2022-01-11T10:15:23.742117Z 2022-01-11T10:15:23.468615Z 2022-01-11T10:15:22.955102Z 2022-01-11T10:15:22.621318Z 2022-01-11T10:15:22.173519Z 2022-01-11T10:15:21.776403Z 2022-01-11T10:15:21.475319Z 2022-01-11T10:15:21.199299Z 2022-01-11T10:15:20.866099Z 2022-01-11T10:15:20.736343Z 2022-01-11T10:15:20.267609Z 2022-01-11T10:15:19.836401Z 2022-01-11T10:15:19.565924Z 2022-01-11T10:15:19.520311Z 2022-01-11T10:15:19.080548Z 2022-01-11T10:15:18.646478Z 2022-01-11T10:15:18.151728Z 2022-01-11T10:15:17.898167Z 2022-01-11T10:15:17.353983Z 2022-01-11T10:15:17.014331Z 2022-01-11T10:15:16.501014Z 2022-01-11T10:15:16.348150Z 2022-01-11T10:15:15.949747Z 2022-01-11T10:15:15.706292Z 2022-01-11T10:15:15.505203Z 2022-01-11T10:15:14.960830Z 2022-01-11T10:15:14.852202Z 2022-01-11T10:15:14.192103Z 2022-01-11T10:15:13.949220Z 2022-01-11T10:15:13.385594Z 2022-01-11T10:15:13.253281Z 2022-01-11T10:15:12.780147Z 2022-01-11T10:15:12.452334Z 2022-01-11T10:15:12.182795Z 2022-01-11T10:15:11.906578Z 2022-01-11T10:15:11.363768Z 2022-01-11T10:15:10.903666Z 2022-01-11T10:15:10.458990Z 2022-01-11T10:15:09.919657Z 2022-01-11T10:15:09.565423Z 2022-01-11T10:15:09.248831Z 2022-01-11T10:15:08.540505Z 2022-01-11T10:15:08.680987Z 2022-01-11T10:15:07.590007Z 2022-01-11T10:15:07.493705Z 2022-01-11T10:15:06.765229Z 2022-01-11T10:15:06.155661Z 2022-01-11T10:15:05.624123Z 2022-01-11T10:15:05.542692Z 2022-01-11T10:15:04.733329Z 2022-01-11T10:15:04.700134Z 2022-01-11T10:15:04.072124Z 2022-01-11T10:15:03.675487Z 2022-01-11T10:15:03.109461Z 2022-01-11T10:15:03.048015Z 2022-01-11T10:15:02.333934Z 2022-01-11T10:15:02.172965Z 2022-01-11T10:15:01.656099Z 2022-01-11T10:15:01.407291Z 2022-01-11T10:15:01.042525Z 2022-01-11T10:15:00.582530Z 2022-01-11T10:15:00.261316Z 2022-01-11T10:14:59.743256Z 2022-01-11T10:14:59.281162Z 2022-01-11T10:14:58.910240Z 2022-01-11T10:14:58.396673Z 2022-01-11T10:14:58.148300Z 2022-01-11T10:14:57.855127Z 2022-01-11T10:14:57.670966Z 2022-01-11T10:14:57.463325Z 2022-01-11T10:14:57.167704Z 2022-01-11T10:14:46.453711Z 2022-01-11T10:14:56.969555Z 2022-01-11T10:14:56.829963Z 2022-01-11T10:14:56.625413Z 2022-01-11T10:14:46.238368Z 2022-01-11T10:14:45.700109Z 2022-01-11T10:14:45.597498Z 2022-01-11T10:14:45.746039Z 2022-01-11T10:14:45.165399Z 2022-01-11T10:14:44.837497Z 2022-01-11T10:14:44.306399Z 2022-01-11T10:14:44.880906Z 2022-01-11T10:14:43.916152Z 2022-01-11T10:14:43.151793Z 2022-01-11T10:14:42.637416Z 2022-01-11T10:14:42.995714Z 2022-01-11T10:14:42.233943Z 2022-01-11T10:14:34.917921Z 2022-01-11T10:14:34.370340Z 2022-01-11T10:14:34.082804Z 2022-01-11T10:14:33.657349Z 2022-01-11T10:14:33.435815Z 2022-01-11T10:14:33.316969Z 2022-01-11T10:14:32.532884Z 2022-01-11T10:14:32.270501Z 2022-01-11T10:14:32.113568Z 2022-01-11T10:14:32.064583Z", + "started_diff": 54.0366776, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 72, 71, 70, 69, 68, 66, 65, 63, 62, 61, 60, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 149, 148, 147, 146, 145, 143, 142, 141, 140, 138, 137, 136, 135, 134, 133, 132, 131, 129, 128, 127, 126, 125, 124, 123, 122, 121, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 171, 170, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 271, 270, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 371, 370, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, 471, 470]" + }, + "started": "2022-01-11T10:41:04.087666+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-api.json new file mode 100644 index 00000000..37cc669c --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-api.json @@ -0,0 +1,10 @@ +{ + "ended": null, + "measurements": {}, + "name": null, + "owner": null, + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-11T23:10:05.159779+00:00" +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-chatty.json new file mode 100644 index 00000000..bde22a7f --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-chatty.json @@ -0,0 +1,636 @@ +{ + "ended": "2022-01-11T22:25:25.132073+00:00", + "id": "run-2022-01-11T21_28_43_550_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 14336.316666666666, + "max": 58126.8, + "mean": 32367.748958333334, + "median": 38565.66666666667, + "min": 1785.6, + "non_zero_mean": 32367.748958333334, + "non_zero_median": 38565.66666666667, + "percentile25": 25479.783333333333, + "percentile75": 39816.1, + "percentile90": 41197.32, + "percentile99": 48731.23199999996, + "percentile999": 57187.24320000007, + "range": 56341.200000000004, + "samples": 64, + "stdev": 12422.129175786587, + "sum": 2071535.9333333345 + } + }, + "cpu": { + "system": { + "iqr": 0.25816666666666677, + "max": 1.0540000000000003, + "mean": 0.84984375, + "median": 0.9986666666666668, + "min": 0.03933333333333356, + "non_zero_mean": 0.84984375, + "non_zero_median": 0.9986666666666668, + "percentile25": 0.7603333333333325, + "percentile75": 1.0184999999999993, + "percentile90": 1.031599999999999, + "percentile99": 1.0506400000000033, + "percentile999": 1.0536640000000006, + "range": 1.0146666666666666, + "samples": 64, + "stdev": 0.2779806431840333, + "sum": 54.39000000000001 + }, + "user": { + "iqr": 2.547333333333335, + "max": 6.498666666666686, + "mean": 5.160625, + "median": 6.288666666666668, + "min": 0.3139999999999835, + "non_zero_mean": 5.160625, + "non_zero_median": 6.288666666666668, + "percentile25": 3.8588333333333202, + "percentile75": 6.406166666666655, + "percentile90": 6.446133333333343, + "percentile99": 6.487326666666671, + "percentile999": 6.497532666666685, + "range": 6.184666666666703, + "samples": 64, + "stdev": 1.8860522561357578, + "sum": 330.28000000000003 + } + }, + "entropy_available_bits": { + "iqr": 7.75, + "max": 214.86666666666667, + "mean": 23.064583333333335, + "median": 6.133333333333333, + "min": 0.0, + "non_zero_mean": 32.08985507246377, + "non_zero_median": 7.633333333333333, + "percentile25": 0.0, + "percentile75": 7.75, + "percentile90": 16.953333333333344, + "percentile99": 213.48066666666665, + "percentile999": 214.72806666666668, + "range": 214.86666666666667, + "samples": 64, + "stdev": 57.582628882589084, + "sum": 1476.1333333333334 + }, + "memory": { + "used": { + "iqr": 1399905280.0, + "max": 7382548480.0, + "mean": 6418891840.0, + "median": 6848950272.0, + "min": 3492077568.0, + "non_zero_mean": 6418891840.0, + "non_zero_median": 6848950272.0, + "percentile25": 5695789056.0, + "percentile75": 7095694336.0, + "percentile90": 7245992345.6, + "percentile99": 7371731107.84, + "percentile999": 7381466742.784, + "range": 3890470912.0, + "samples": 64, + "stdev": 899359805.095002, + "sum": 410809077760.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.010416666666666666, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2222222222222222, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.26666666666666666, + "percentile999": 0.26666666666666666, + "range": 0.26666666666666666, + "samples": 64, + "stdev": 0.049289484302299166, + "sum": 0.6666666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 2252.066666666667, + "max": 4868.933333333333, + "mean": 3604.909375, + "median": 4558.966666666667, + "min": 19.599999999999998, + "non_zero_mean": 3604.909375, + "non_zero_median": 4558.966666666667, + "percentile25": 2450.4, + "percentile75": 4702.466666666667, + "percentile90": 4758.426666666666, + "percentile99": 4858.853333333334, + "percentile999": 4867.925333333334, + "range": 4849.333333333333, + "samples": 64, + "stdev": 1530.7712877601919, + "sum": 230714.19999999998 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 191.1166666666668, + "max": 2536.4666666666667, + "mean": 1639.2395833333333, + "median": 1716.6666666666665, + "min": 500.3333333333333, + "non_zero_mean": 1639.2395833333333, + "non_zero_median": 1716.6666666666665, + "percentile25": 1590.8166666666666, + "percentile75": 1781.9333333333334, + "percentile90": 1912.64, + "percentile99": 2283.1646666666657, + "percentile999": 2511.1364666666686, + "range": 2036.1333333333334, + "samples": 64, + "stdev": 345.18293853229756, + "sum": 104911.33333333334 + } + }, + "cpu": { + "system": { + "iqr": 0.03749999999999995, + "max": 0.26733333333333364, + "mean": 0.14436458333333332, + "median": 0.15533333333333346, + "min": 0.0206666666666667, + "non_zero_mean": 0.14436458333333332, + "non_zero_median": 0.15533333333333346, + "percentile25": 0.12983333333333327, + "percentile75": 0.16733333333333322, + "percentile90": 0.1811333333333331, + "percentile99": 0.2488533333333334, + "percentile999": 0.26548533333333374, + "range": 0.24666666666666695, + "samples": 64, + "stdev": 0.04650052506767709, + "sum": 9.239333333333333 + }, + "user": { + "iqr": 0.041833333333334055, + "max": 0.5640000000000001, + "mean": 0.21620833333333334, + "median": 0.17833333333333312, + "min": 0.028666666666667125, + "non_zero_mean": 0.21620833333333334, + "non_zero_median": 0.17833333333333312, + "percentile25": 0.1631666666666664, + "percentile75": 0.20500000000000046, + "percentile90": 0.4982666666666653, + "percentile99": 0.5530800000000001, + "percentile999": 0.5629080000000002, + "range": 0.5353333333333329, + "samples": 64, + "stdev": 0.1358405993447286, + "sum": 13.837333333333335 + } + }, + "entropy_available_bits": { + "iqr": 1.5333333333333334, + "max": 187.06666666666666, + "mean": 9.897916666666667, + "median": 2.3, + "min": 0.4666666666666667, + "non_zero_mean": 9.897916666666667, + "non_zero_median": 2.3, + "percentile25": 1.3333333333333333, + "percentile75": 2.8666666666666667, + "percentile90": 4.146666666666667, + "percentile99": 177.23866666666663, + "percentile999": 186.08386666666675, + "range": 186.6, + "samples": 64, + "stdev": 35.14015870721494, + "sum": 633.4666666666666 + }, + "memory": { + "used": { + "iqr": 34864128.0, + "max": 477532160.0, + "mean": 445678528.0, + "median": 458141696.0, + "min": 368861184.0, + "non_zero_mean": 445678528.0, + "non_zero_median": 458141696.0, + "percentile25": 431282176.0, + "percentile75": 466146304.0, + "percentile90": 472147968.0, + "percentile99": 476807045.12, + "percentile999": 477459648.512, + "range": 108670976.0, + "samples": 64, + "stdev": 30337814.79186938, + "sum": 28523425792.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1914470.4, + "max": 3413879.466666667, + "mean": 806237.8666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1984585.5179487178, + "non_zero_median": 2111897.6, + "percentile25": 0.0, + "percentile75": 1914470.4, + "percentile90": 2969654.613333334, + "percentile99": 3407342.250666667, + "percentile999": 3413225.745066667, + "range": 3413879.466666667, + "samples": 64, + "stdev": 1198106.324254976, + "sum": 51599223.46666667 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0067095, + "max": 0.271602, + "mean": 0.010594640625, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.03568721052631579, + "non_zero_median": 0.015748, + "percentile25": 0.0, + "percentile75": 0.0067095, + "percentile90": 0.027266700000000015, + "percentile99": 0.16204373999999955, + "percentile999": 0.26064617400000084, + "range": 0.271602, + "samples": 64, + "stdev": 0.03654671141280734, + "sum": 0.6780569999999999 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 1.467634, + "mean": 0.03502365625, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2241514, + "non_zero_median": 0.021054999999999997, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.014232900000000003, + "percentile99": 0.8519646099999976, + "percentile999": 1.4060670610000048, + "range": 1.467634, + "samples": 64, + "stdev": 0.1929964860274905, + "sum": 2.2415140000000005 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 25.6, + "mean": 0.6947916666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 14.822222222222223, + "non_zero_median": 11.066666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 16.443999999999964, + "percentile999": 24.68440000000007, + "range": 25.6, + "samples": 64, + "stdev": 3.5807599655962603, + "sum": 44.46666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9983.733333333334, + "mean": 467.49375, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9973.2, + "non_zero_median": 9968.333333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9974.031333333334, + "percentile999": 9982.763133333334, + "range": 9983.733333333334, + "samples": 64, + "stdev": 2124.7113546788087, + "sum": 29919.600000000002 + }, + "pg_stat_database_blks_hit": { + "iqr": 2996.4833333333336, + "max": 40084.333333333336, + "mean": 21739.170833333334, + "median": 22875.3, + "min": 4210.933333333333, + "non_zero_mean": 21739.170833333334, + "non_zero_median": 22875.3, + "percentile25": 20898.616666666665, + "percentile75": 23895.1, + "percentile90": 25274.466666666667, + "percentile99": 39286.54333333333, + "percentile999": 40004.55433333334, + "range": 35873.4, + "samples": 64, + "stdev": 6262.729769791734, + "sum": 1391306.9333333333 + }, + "pg_stat_database_blks_read": { + "iqr": 70.35000000000002, + "max": 150.66666666666666, + "mean": 100.52604166666667, + "median": 131.33333333333334, + "min": 0.0, + "non_zero_mean": 102.12169312169313, + "non_zero_median": 131.86666666666667, + "percentile25": 65.5, + "percentile75": 135.85000000000002, + "percentile90": 139.26666666666668, + "percentile99": 147.51666666666665, + "percentile999": 150.3516666666667, + "range": 150.66666666666666, + "samples": 64, + "stdev": 49.75255471669513, + "sum": 6433.666666666665 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 70.35000000000002, + "max": 150.66666666666666, + "mean": 100.52604166666667, + "median": 131.33333333333334, + "min": 0.0, + "non_zero_mean": 102.12169312169313, + "non_zero_median": 131.86666666666667, + "percentile25": 65.5, + "percentile75": 135.85000000000002, + "percentile90": 139.26666666666668, + "percentile99": 147.51666666666665, + "percentile999": 150.3516666666667, + "range": 150.66666666666666, + "samples": 64, + "stdev": 49.75255471669513, + "sum": 6433.666666666665 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.03333333333333333, + "max": 1.2, + "mean": 0.18854166666666666, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7541666666666667, + "non_zero_median": 0.8666666666666667, + "percentile25": 0.0, + "percentile75": 0.03333333333333333, + "percentile90": 0.9800000000000002, + "percentile99": 1.1159999999999997, + "percentile999": 1.1916000000000007, + "range": 1.2, + "samples": 64, + "stdev": 0.36909817882030466, + "sum": 12.066666666666665 + }, + "pg_stat_database_tup_fetched": { + "iqr": 2014.9499999999998, + "max": 25238.533333333333, + "mean": 7134.709375, + "median": 5300.866666666667, + "min": 2172.5333333333333, + "non_zero_mean": 7134.709375, + "non_zero_median": 5300.866666666667, + "percentile25": 4896.7, + "percentile75": 6911.65, + "percentile90": 11999.713333333333, + "percentile99": 24084.12133333333, + "percentile999": 25123.092133333343, + "range": 23066.0, + "samples": 64, + "stdev": 4525.111205545806, + "sum": 456621.4000000001 + }, + "pg_stat_database_tup_inserted": { + "iqr": 364.0666666666667, + "max": 799.4, + "mean": 534.5041666666667, + "median": 694.0999999999999, + "min": 0.0, + "non_zero_mean": 542.9883597883598, + "non_zero_median": 701.6666666666666, + "percentile25": 355.16666666666663, + "percentile75": 719.2333333333333, + "percentile90": 742.9133333333334, + "percentile99": 794.6959999999999, + "percentile999": 798.9296, + "range": 799.4, + "samples": 64, + "stdev": 262.7754419507581, + "sum": 34208.266666666656 + }, + "pg_stat_database_tup_returned": { + "iqr": 1792.5166666666673, + "max": 29887.133333333335, + "mean": 8976.644791666668, + "median": 6595.766666666666, + "min": 3301.8, + "non_zero_mean": 8976.644791666668, + "non_zero_median": 6595.766666666666, + "percentile25": 6162.183333333333, + "percentile75": 7954.700000000001, + "percentile90": 17151.58666666667, + "percentile99": 29095.09733333333, + "percentile999": 29807.92973333334, + "range": 26585.333333333336, + "samples": 64, + "stdev": 5767.419766294338, + "sum": 574505.2666666666 + }, + "pg_stat_database_tup_updated": { + "iqr": 8.016666666666667, + "max": 137.06666666666666, + "mean": 10.166666666666666, + "median": 0.6666666666666666, + "min": 0.0, + "non_zero_mean": 11.83030303030303, + "non_zero_median": 1.2666666666666666, + "percentile25": 0.18333333333333335, + "percentile75": 8.200000000000001, + "percentile90": 24.60666666666669, + "percentile99": 119.8046666666666, + "percentile999": 135.3404666666668, + "range": 137.06666666666666, + "samples": 64, + "stdev": 26.13292840632192, + "sum": 650.6666666666665 + }, + "pg_stat_database_xact_commit": { + "iqr": 12.799999999999997, + "max": 149.26666666666668, + "mean": 59.166666666666664, + "median": 50.5, + "min": 8.266666666666667, + "non_zero_mean": 59.166666666666664, + "non_zero_median": 50.5, + "percentile25": 47.1, + "percentile75": 59.9, + "percentile90": 104.4666666666667, + "percentile99": 140.65666666666664, + "percentile999": 148.40566666666675, + "range": 141.0, + "samples": 64, + "stdev": 28.449156355760312, + "sum": 3786.666666666668 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 4.266666666666671 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.004166666666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.09866666666666599, + "percentile999": 0.24986666666666793, + "range": 0.26666666666666666, + "samples": 64, + "stdev": 0.03333333333333333, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 22.33333333333333, + "max": 166.2, + "mean": 78.29895833333333, + "median": 80.8, + "min": 3.8666666666666667, + "non_zero_mean": 78.29895833333333, + "non_zero_median": 80.8, + "percentile25": 69.03333333333333, + "percentile75": 91.36666666666666, + "percentile90": 106.42666666666668, + "percentile99": 140.91599999999988, + "percentile999": 163.67160000000018, + "range": 162.33333333333331, + "samples": 64, + "stdev": 27.83249466071629, + "sum": 5011.133333333333 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "4,3,6,5,8,9,7,11,10,14,18,19,25,21,20,22,23,24,26,27,29,28,30,31,32,34,33,35,36,37,38,40,42,43,41,39,44,45,46,47,48,49,52,51,54,50,53,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-11T22:06:50.230386Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2022-01-11T22:06:50.926904Z 2022-01-11T22:06:50.926708Z 2022-01-11T22:06:50.879338Z 2022-01-11T22:06:50.362325Z 2022-01-11T22:06:50.287896Z 2022-01-11T22:06:50.242093Z 2022-01-11T22:06:50.230386Z 2022-01-11T22:06:52.034556Z 2022-01-11T22:06:51.045531Z 2022-01-11T22:06:51.008772Z 2022-01-11T22:10:56.453060Z 2022-01-11T22:10:56.038360Z 2022-01-11T22:10:55.630179Z 2022-01-11T22:10:55.626857Z 2022-01-11T22:10:55.600480Z 2022-01-11T22:10:55.564057Z 2022-01-11T22:10:55.550793Z 2022-01-11T22:10:55.536896Z 2022-01-11T22:10:55.433864Z 2022-01-11T22:10:55.428557Z 2022-01-11T22:14:02.104426Z 2022-01-11T22:14:01.916934Z 2022-01-11T22:14:01.468294Z 2022-01-11T22:14:01.468360Z 2022-01-11T22:14:01.464099Z 2022-01-11T22:14:01.412999Z 2022-01-11T22:14:01.402489Z 2022-01-11T22:14:01.350430Z 2022-01-11T22:14:01.339960Z 2022-01-11T22:14:01.339899Z 2022-01-11T22:17:06.436972Z 2022-01-11T22:17:06.405804Z 2022-01-11T22:17:05.927625Z 2022-01-11T22:17:05.856635Z 2022-01-11T22:17:05.853597Z 2022-01-11T22:17:05.851898Z 2022-01-11T22:17:05.847190Z 2022-01-11T22:17:05.844837Z 2022-01-11T22:17:05.837038Z 2022-01-11T22:17:05.812006Z 2022-01-11T22:20:00.498041Z 2022-01-11T22:20:00.315236Z 2022-01-11T22:19:59.850806Z 2022-01-11T22:19:59.801733Z 2022-01-11T22:19:59.793133Z 2022-01-11T22:19:59.747038Z 2022-01-11T22:19:59.739594Z 2022-01-11T22:19:59.735208Z 2022-01-11T22:19:59.652970Z 2022-01-11T22:19:59.637890Z", + "created_diff": 1.0156634, + "ended": "2022-01-11T22:22:43.060531Z", + "error_job_ids": " []", + "event_first": "2022-01-11T22:07:01.765611Z 2022-01-11T22:07:52.108715Z 2022-01-11T22:07:00.308034Z 2022-01-11T22:08:20.704455Z 2022-01-11T22:06:56.873636Z 2022-01-11T22:07:41.646173Z 2022-01-11T22:06:56.361124Z 2022-01-11T22:08:04.227711Z 2022-01-11T22:07:06.890391Z 2022-01-11T22:08:38.837153Z 2022-01-11T22:11:11.837209Z 2022-01-11T22:11:12.859876Z 2022-01-11T22:11:05.266867Z 2022-01-11T22:11:05.636546Z 2022-01-11T22:11:04.080635Z 2022-01-11T22:11:04.099680Z 2022-01-11T22:11:03.264196Z 2022-01-11T22:11:03.339521Z 2022-01-11T22:11:01.713003Z 2022-01-11T22:11:02.585561Z 2022-01-11T22:14:21.272431Z 2022-01-11T22:14:20.387417Z 2022-01-11T22:14:13.253453Z 2022-01-11T22:14:10.844699Z 2022-01-11T22:14:08.600454Z 2022-01-11T22:14:08.064997Z 2022-01-11T22:14:08.555962Z 2022-01-11T22:14:08.064997Z 2022-01-11T22:14:07.361087Z 2022-01-11T22:14:07.326947Z 2022-01-11T22:17:22.984944Z 2022-01-11T22:17:23.605862Z 2022-01-11T22:17:16.819794Z 2022-01-11T22:17:15.531530Z 2022-01-11T22:17:13.575225Z 2022-01-11T22:17:13.790513Z 2022-01-11T22:17:13.308843Z 2022-01-11T22:17:13.790513Z 2022-01-11T22:17:12.826446Z 2022-01-11T22:17:12.265243Z 2022-01-11T22:20:22.708295Z 2022-01-11T22:20:22.750503Z 2022-01-11T22:20:06.989704Z 2022-01-11T22:20:10.054827Z 2022-01-11T22:20:09.365704Z 2022-01-11T22:20:07.234668Z 2022-01-11T22:20:06.731681Z 2022-01-11T22:20:07.234668Z 2022-01-11T22:20:05.724084Z 2022-01-11T22:20:06.048201Z", + "event_last": "2022-01-11T22:09:35.098602Z 2022-01-11T22:10:15.169787Z 2022-01-11T22:09:34.454953Z 2022-01-11T22:10:33.402758Z 2022-01-11T22:09:29.928972Z 2022-01-11T22:10:06.532829Z 2022-01-11T22:09:29.083372Z 2022-01-11T22:10:22.445018Z 2022-01-11T22:09:40.932395Z 2022-01-11T22:10:44.355781Z 2022-01-11T22:13:34.985180Z 2022-01-11T22:13:34.561715Z 2022-01-11T22:13:30.028701Z 2022-01-11T22:13:32.390292Z 2022-01-11T22:13:28.432860Z 2022-01-11T22:13:30.391657Z 2022-01-11T22:13:29.500700Z 2022-01-11T22:13:31.635257Z 2022-01-11T22:13:25.938287Z 2022-01-11T22:13:30.896404Z 2022-01-11T22:16:40.695502Z 2022-01-11T22:16:42.308539Z 2022-01-11T22:16:35.429135Z 2022-01-11T22:16:35.350048Z 2022-01-11T22:16:33.837237Z 2022-01-11T22:16:30.651304Z 2022-01-11T22:16:31.777088Z 2022-01-11T22:16:31.635976Z 2022-01-11T22:16:32.240061Z 2022-01-11T22:16:32.939518Z 2022-01-11T22:19:41.744845Z 2022-01-11T22:19:42.428234Z 2022-01-11T22:19:38.711083Z 2022-01-11T22:19:38.773409Z 2022-01-11T22:19:37.869529Z 2022-01-11T22:19:37.469410Z 2022-01-11T22:19:34.913556Z 2022-01-11T22:19:37.469410Z 2022-01-11T22:19:33.962591Z 2022-01-11T22:19:35.455368Z 2022-01-11T22:22:43.060531Z 2022-01-11T22:22:41.429725Z 2022-01-11T22:22:31.312688Z 2022-01-11T22:22:33.633816Z 2022-01-11T22:22:32.250294Z 2022-01-11T22:22:31.284442Z 2022-01-11T22:22:29.456435Z 2022-01-11T22:22:32.381820Z 2022-01-11T22:22:29.793235Z 2022-01-11T22:22:31.512444Z", + "failed_job_ids": " []", + "finished": "2022-01-11T22:09:39.037846Z 2022-01-11T22:10:18.948096Z 2022-01-11T22:09:38.406961Z 2022-01-11T22:10:37.490565Z 2022-01-11T22:09:34.198453Z 2022-01-11T22:10:10.705131Z 2022-01-11T22:09:33.056640Z 2022-01-11T22:10:26.258931Z 2022-01-11T22:09:45.170626Z 2022-01-11T22:10:48.414128Z 2022-01-11T22:13:37.418338Z 2022-01-11T22:13:38.392654Z 2022-01-11T22:13:34.143252Z 2022-01-11T22:13:36.223182Z 2022-01-11T22:13:32.701539Z 2022-01-11T22:13:34.787394Z 2022-01-11T22:13:33.392595Z 2022-01-11T22:13:35.770842Z 2022-01-11T22:13:29.205387Z 2022-01-11T22:13:34.397635Z 2022-01-11T22:16:44.399840Z 2022-01-11T22:16:43.714941Z 2022-01-11T22:16:38.911598Z 2022-01-11T22:16:39.165040Z 2022-01-11T22:16:37.690980Z 2022-01-11T22:16:34.317773Z 2022-01-11T22:16:35.309065Z 2022-01-11T22:16:35.640476Z 2022-01-11T22:16:36.215250Z 2022-01-11T22:16:36.287238Z 2022-01-11T22:19:45.272108Z 2022-01-11T22:19:46.076071Z 2022-01-11T22:19:42.042515Z 2022-01-11T22:19:42.594015Z 2022-01-11T22:19:41.296002Z 2022-01-11T22:19:41.290815Z 2022-01-11T22:19:38.941929Z 2022-01-11T22:19:41.127192Z 2022-01-11T22:19:37.493605Z 2022-01-11T22:19:39.553645Z 2022-01-11T22:22:45.486716Z 2022-01-11T22:22:45.111386Z 2022-01-11T22:22:35.502159Z 2022-01-11T22:22:37.425720Z 2022-01-11T22:22:36.026773Z 2022-01-11T22:22:35.060722Z 2022-01-11T22:22:32.737066Z 2022-01-11T22:22:36.224653Z 2022-01-11T22:22:33.445704Z 2022-01-11T22:22:35.508179Z", + "finished_diff": 23.1917876, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 235.764427, + "mean": 161.72404913999995, + "min": 150.554836 + }, + "jobs_events_duration": { + "max": 154.146919, + "mean": 143.80926007999997, + "min": 125.518628 + }, + "jobs_events_lag": { + "max": -1.406402, + "mean": -3.7209315599999995, + "min": -4.395737 + }, + "jobs_waiting": { + "max": 13.571476, + "mean": 2.46532322, + "min": 0.925799 + }, + "modified": "2022-01-11T22:06:52.100229Z 2022-01-11T22:06:52.059097Z 2022-01-11T22:06:52.018798Z 2022-01-11T22:06:50.958049Z 2022-01-11T22:06:50.924906Z 2022-01-11T22:06:50.888354Z 2022-01-11T22:06:50.849477Z 2022-01-11T22:07:05.296586Z 2022-01-11T22:06:52.178006Z 2022-01-11T22:06:52.136392Z 2022-01-11T22:10:57.872440Z 2022-01-11T22:10:57.834539Z 2022-01-11T22:10:56.230754Z 2022-01-11T22:10:56.200612Z 2022-01-11T22:10:56.169745Z 2022-01-11T22:10:56.133636Z 2022-01-11T22:10:56.099006Z 2022-01-11T22:10:56.064500Z 2022-01-11T22:10:56.032540Z 2022-01-11T22:10:55.999729Z 2022-01-11T22:14:08.972227Z 2022-01-11T22:14:08.899416Z 2022-01-11T22:14:02.089481Z 2022-01-11T22:14:02.121624Z 2022-01-11T22:14:02.052898Z 2022-01-11T22:14:02.018589Z 2022-01-11T22:14:01.984669Z 2022-01-11T22:14:01.951256Z 2022-01-11T22:14:01.918679Z 2022-01-11T22:14:01.883827Z 2022-01-11T22:17:10.666683Z 2022-01-11T22:17:10.631674Z 2022-01-11T22:17:06.604192Z 2022-01-11T22:17:06.571633Z 2022-01-11T22:17:06.537781Z 2022-01-11T22:17:06.503981Z 2022-01-11T22:17:06.471727Z 2022-01-11T22:17:06.441690Z 2022-01-11T22:17:06.412522Z 2022-01-11T22:17:06.382226Z 2022-01-11T22:20:13.005845Z 2022-01-11T22:20:12.911621Z 2022-01-11T22:20:00.510628Z 2022-01-11T22:20:00.480784Z 2022-01-11T22:20:00.451136Z 2022-01-11T22:20:00.418692Z 2022-01-11T22:20:00.385141Z 2022-01-11T22:20:00.354694Z 2022-01-11T22:20:00.323950Z 2022-01-11T22:20:00.291173Z", + "modified_diff": 8.081469799999999, + "started": "2022-01-11T22:06:52.608852Z 2022-01-11T22:06:52.495650Z 2022-01-11T22:06:52.372624Z 2022-01-11T22:06:51.461977Z 2022-01-11T22:06:51.327058Z 2022-01-11T22:06:51.302345Z 2022-01-11T22:06:51.156185Z 2022-01-11T22:07:05.606032Z 2022-01-11T22:06:52.785662Z 2022-01-11T22:06:52.649701Z 2022-01-11T22:10:58.362688Z 2022-01-11T22:10:58.302355Z 2022-01-11T22:10:57.090090Z 2022-01-11T22:10:57.026558Z 2022-01-11T22:10:56.887266Z 2022-01-11T22:10:56.831905Z 2022-01-11T22:10:56.709578Z 2022-01-11T22:10:56.677285Z 2022-01-11T22:10:56.560502Z 2022-01-11T22:10:56.486918Z 2022-01-11T22:14:10.009545Z 2022-01-11T22:14:10.005406Z 2022-01-11T22:14:02.862513Z 2022-01-11T22:14:02.999567Z 2022-01-11T22:14:02.791186Z 2022-01-11T22:14:02.705694Z 2022-01-11T22:14:02.612800Z 2022-01-11T22:14:02.550968Z 2022-01-11T22:14:02.453865Z 2022-01-11T22:14:02.378950Z 2022-01-11T22:17:11.182039Z 2022-01-11T22:17:11.035236Z 2022-01-11T22:17:07.589243Z 2022-01-11T22:17:07.477084Z 2022-01-11T22:17:07.302550Z 2022-01-11T22:17:07.224697Z 2022-01-11T22:17:07.101717Z 2022-01-11T22:17:07.023598Z 2022-01-11T22:17:06.938769Z 2022-01-11T22:17:06.863824Z 2022-01-11T22:20:13.802773Z 2022-01-11T22:20:13.581644Z 2022-01-11T22:20:01.437249Z 2022-01-11T22:20:01.322100Z 2022-01-11T22:20:01.183902Z 2022-01-11T22:20:01.092215Z 2022-01-11T22:20:01.011105Z 2022-01-11T22:20:00.943940Z 2022-01-11T22:20:00.840583Z 2022-01-11T22:20:00.760921Z", + "started_diff": 8.2632558, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-11T22:25:22.647761+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-launching.json new file mode 100644 index 00000000..ea4f9ead --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-11T21_28_43_550_0000-launching.json @@ -0,0 +1,634 @@ +{ + "ended": "2022-01-11T23:09:18.552316+00:00", + "id": "run-2022-01-11T21_28_43_550_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17546.733333333334, + "max": 26931.466666666667, + "mean": 10270.472222222223, + "median": 4444.299999999999, + "min": 1501.1999999999998, + "non_zero_mean": 10270.472222222223, + "non_zero_median": 4444.299999999999, + "percentile25": 1804.6, + "percentile75": 19351.333333333332, + "percentile90": 23527.173333333332, + "percentile99": 26197.989333333342, + "percentile999": 26858.118933333335, + "range": 25430.266666666666, + "samples": 72, + "stdev": 9227.606889180528, + "sum": 739474.0 + } + }, + "cpu": { + "system": { + "iqr": 1.4296666666666666, + "max": 1.8180000000000005, + "mean": 0.7113981481481482, + "median": 0.25966666666666927, + "min": 0.026666666666667047, + "non_zero_mean": 0.7113981481481482, + "non_zero_median": 0.25966666666666927, + "percentile25": 0.03916666666666894, + "percentile75": 1.4688333333333357, + "percentile90": 1.6165333333333314, + "percentile99": 1.740373333333334, + "percentile999": 1.810237333333334, + "range": 1.7913333333333334, + "samples": 72, + "stdev": 0.7003220467297901, + "sum": 51.22066666666666 + }, + "user": { + "iqr": 5.8188333333333295, + "max": 6.494000000000005, + "mean": 3.041777777777778, + "median": 2.2196666666666793, + "min": 0.08333333333331819, + "non_zero_mean": 3.041777777777778, + "non_zero_median": 2.2196666666666793, + "percentile25": 0.33500000000000607, + "percentile75": 6.153833333333336, + "percentile90": 6.315733333333335, + "percentile99": 6.4495066666666805, + "percentile999": 6.4895506666666725, + "range": 6.410666666666687, + "samples": 72, + "stdev": 2.7738355404806905, + "sum": 219.00800000000004 + } + }, + "entropy_available_bits": { + "iqr": 4.666666666666667, + "max": 421.9333333333333, + "mean": 25.33425925925926, + "median": 0.7666666666666667, + "min": 0.0, + "non_zero_mean": 34.41635220125786, + "non_zero_median": 3.6, + "percentile25": 0.0, + "percentile75": 4.666666666666667, + "percentile90": 5.726666666666666, + "percentile99": 421.8386666666666, + "percentile999": 421.92386666666664, + "range": 421.9333333333333, + "samples": 72, + "stdev": 96.77023656360768, + "sum": 1824.0666666666662 + }, + "memory": { + "used": { + "iqr": 5734446080.0, + "max": 16837373952.0, + "mean": 8541867235.555555, + "median": 6181470208.0, + "min": 5250363392.0, + "non_zero_mean": 8541867235.555555, + "non_zero_median": 6181470208.0, + "percentile25": 5920235520.0, + "percentile75": 11654681600.0, + "percentile90": 14896265216.0, + "percentile99": 16283663196.160006, + "percentile999": 16782002876.416002, + "range": 11587010560.0, + "samples": 72, + "stdev": 3610316098.1218915, + "sum": 615014440960.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003703703703703704, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.077333333333335, + "percentile999": 0.24773333333333386, + "range": 0.26666666666666666, + "samples": 72, + "stdev": 0.03142696805273544, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 899.8500000000001, + "max": 1778.8666666666666, + "mean": 481.65185185185186, + "median": 65.43333333333334, + "min": 0.13333333333333333, + "non_zero_mean": 481.65185185185186, + "non_zero_median": 65.43333333333334, + "percentile25": 3.783333333333333, + "percentile75": 903.6333333333334, + "percentile90": 1433.7066666666667, + "percentile99": 1679.7980000000007, + "percentile999": 1768.9598, + "range": 1778.7333333333331, + "samples": 72, + "stdev": 581.2618102426827, + "sum": 34678.933333333334 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2299.6666666666665, + "max": 3638.2, + "mean": 1787.4268518518518, + "median": 2132.5333333333333, + "min": 341.4, + "non_zero_mean": 1787.4268518518518, + "non_zero_median": 2132.5333333333333, + "percentile25": 616.2833333333333, + "percentile75": 2915.95, + "percentile90": 3282.4133333333334, + "percentile99": 3628.686, + "percentile999": 3637.2486, + "range": 3296.7999999999997, + "samples": 72, + "stdev": 1146.1067840282526, + "sum": 128694.73333333335 + } + }, + "cpu": { + "system": { + "iqr": 0.23149999999999976, + "max": 0.38599999999999945, + "mean": 0.13955555555555557, + "median": 0.13000000000000042, + "min": 0.00933333333333337, + "non_zero_mean": 0.13955555555555557, + "non_zero_median": 0.13000000000000042, + "percentile25": 0.023999999999999962, + "percentile75": 0.2554999999999997, + "percentile90": 0.3098666666666673, + "percentile99": 0.37653333333333344, + "percentile999": 0.38505333333333286, + "range": 0.3766666666666661, + "samples": 72, + "stdev": 0.12105679676837001, + "sum": 10.048000000000004 + }, + "user": { + "iqr": 0.3114999999999981, + "max": 0.727333333333332, + "mean": 0.2127685185185185, + "median": 0.1849999999999985, + "min": 0.010666666666670228, + "non_zero_mean": 0.2127685185185185, + "non_zero_median": 0.1849999999999985, + "percentile25": 0.040000000000001285, + "percentile75": 0.35149999999999937, + "percentile90": 0.455600000000001, + "percentile99": 0.7121866666666665, + "percentile999": 0.7258186666666655, + "range": 0.7166666666666618, + "samples": 72, + "stdev": 0.19627634250115392, + "sum": 15.319333333333333 + } + }, + "entropy_available_bits": { + "iqr": 3.283333333333333, + "max": 214.0, + "mean": 12.881481481481481, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 26.49904761904762, + "non_zero_median": 3.3333333333333335, + "percentile25": 0.0, + "percentile75": 3.283333333333333, + "percentile90": 4.446666666666666, + "percentile99": 213.95266666666666, + "percentile999": 213.99526666666668, + "range": 214.0, + "samples": 72, + "stdev": 48.61138054921975, + "sum": 927.4666666666667 + }, + "memory": { + "used": { + "iqr": 407707648.0, + "max": 1105965056.0, + "mean": 579604992.0, + "median": 425529344.0, + "min": 368082944.0, + "non_zero_mean": 579604992.0, + "non_zero_median": 425529344.0, + "percentile25": 375755776.0, + "percentile75": 783463424.0, + "percentile90": 989254451.2, + "percentile99": 1072204226.5600003, + "percentile999": 1102588973.056, + "range": 737882112.0, + "samples": 72, + "stdev": 250509871.32671282, + "sum": 41731559424.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 5188.266666666666, + "max": 183500.8, + "mean": 19273.955555555556, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 69386.24, + "non_zero_median": 59528.53333333333, + "percentile25": 0.0, + "percentile75": 5188.266666666666, + "percentile90": 73782.61333333333, + "percentile99": 166439.59466666682, + "percentile999": 181794.6794666667, + "range": 183500.8, + "samples": 72, + "stdev": 41932.6141658775, + "sum": 1387724.7999999998 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 2.168129, + "mean": 0.04887877777777778, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.2932726666666667, + "non_zero_median": 0.036644499999999997, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.02604799999999999, + "percentile99": 1.04628711000001, + "percentile999": 2.0559448110000034, + "range": 2.168129, + "samples": 72, + "stdev": 0.2676845332407461, + "sum": 3.519272 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.2277155, + "max": 7.171323, + "mean": 0.4664826388888889, + "median": 0.0, + "min": -0.001339, + "non_zero_mean": 0.9596214285714286, + "non_zero_median": 0.264797, + "percentile25": 0.0, + "percentile75": 0.2277155, + "percentile90": 1.3746081, + "percentile99": 5.039437240000019, + "percentile999": 6.958134424000006, + "range": 7.172662, + "samples": 72, + "stdev": 1.1370301551771393, + "sum": 33.58675 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.2, + "mean": 0.11666666666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.8000000000000003, + "non_zero_median": 3.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.058000000000001, + "percentile999": 3.1858000000000004, + "range": 3.2, + "samples": 72, + "stdev": 0.5703964521944859, + "sum": 8.4 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9996.133333333333, + "mean": 382.40092592592595, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9177.622222222222, + "non_zero_median": 9989.866666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9991.684, + "percentile999": 9995.6884, + "range": 9996.133333333333, + "samples": 72, + "stdev": 1861.9493057246034, + "sum": 27532.86666666667 + }, + "pg_stat_database_blks_hit": { + "iqr": 24460.950000000004, + "max": 100590.66666666667, + "mean": 23429.390740740742, + "median": 17474.033333333333, + "min": 1426.3333333333333, + "non_zero_mean": 23429.390740740742, + "non_zero_median": 17474.033333333333, + "percentile25": 7684.65, + "percentile75": 32145.600000000002, + "percentile90": 48425.806666666664, + "percentile99": 95847.25133333338, + "percentile999": 100116.32513333335, + "range": 99164.33333333334, + "samples": 72, + "stdev": 22222.06411445656, + "sum": 1686916.133333333 + }, + "pg_stat_database_blks_read": { + "iqr": 4.033333333333333, + "max": 13.266666666666667, + "mean": 2.3555555555555556, + "median": 0.7333333333333334, + "min": 0.0, + "non_zero_mean": 3.7688888888888887, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 4.033333333333333, + "percentile90": 6.526666666666666, + "percentile99": 12.746000000000004, + "percentile999": 13.214600000000003, + "range": 13.266666666666667, + "samples": 72, + "stdev": 3.318690844322169, + "sum": 169.59999999999997 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 72, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 4.033333333333333, + "max": 13.266666666666667, + "mean": 2.3555555555555556, + "median": 0.7333333333333334, + "min": 0.0, + "non_zero_mean": 3.7688888888888887, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 4.033333333333333, + "percentile90": 6.526666666666666, + "percentile99": 12.746000000000004, + "percentile999": 13.214600000000003, + "range": 13.266666666666667, + "samples": 72, + "stdev": 3.318690844322169, + "sum": 169.59999999999997 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4666666666666666, + "mean": 0.15555555555555556, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.018181818181818, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.8666666666666667, + "percentile99": 1.3246666666666678, + "percentile999": 1.452466666666667, + "range": 1.4666666666666666, + "samples": 72, + "stdev": 0.379770130622033, + "sum": 11.200000000000001 + }, + "pg_stat_database_tup_fetched": { + "iqr": 13744.333333333334, + "max": 50326.933333333334, + "mean": 12476.803703703703, + "median": 9407.7, + "min": 708.8, + "non_zero_mean": 12476.803703703703, + "non_zero_median": 9407.7, + "percentile25": 3927.1, + "percentile75": 17671.433333333334, + "percentile90": 26838.11333333333, + "percentile99": 48349.394000000015, + "percentile999": 50129.17940000001, + "range": 49618.13333333333, + "samples": 72, + "stdev": 11509.55784758898, + "sum": 898329.8666666667 + }, + "pg_stat_database_tup_inserted": { + "iqr": 32.233333333333334, + "max": 100.0, + "mean": 18.233333333333334, + "median": 5.133333333333333, + "min": 0.0, + "non_zero_mean": 28.53913043478261, + "non_zero_median": 23.366666666666667, + "percentile25": 0.0, + "percentile75": 32.233333333333334, + "percentile90": 49.43999999999999, + "percentile99": 98.95866666666667, + "percentile999": 99.89586666666666, + "range": 100.0, + "samples": 72, + "stdev": 24.766252468749745, + "sum": 1312.8000000000002 + }, + "pg_stat_database_tup_returned": { + "iqr": 18923.88333333333, + "max": 76044.33333333333, + "mean": 18491.937962962962, + "median": 14549.766666666666, + "min": 1902.6, + "non_zero_mean": 18491.937962962962, + "non_zero_median": 14549.766666666666, + "percentile25": 6802.616666666667, + "percentile75": 25726.5, + "percentile90": 35905.61333333333, + "percentile99": 75798.29466666667, + "percentile999": 76019.72946666666, + "range": 74141.73333333332, + "samples": 72, + "stdev": 16903.62125625754, + "sum": 1331419.5333333332 + }, + "pg_stat_database_tup_updated": { + "iqr": 32.0, + "max": 72.6, + "mean": 18.35648148148148, + "median": 14.066666666666666, + "min": 0.0, + "non_zero_mean": 20.978835978835978, + "non_zero_median": 20.333333333333332, + "percentile25": 0.26666666666666666, + "percentile75": 32.266666666666666, + "percentile90": 49.379999999999995, + "percentile99": 67.06200000000004, + "percentile999": 72.04620000000001, + "range": 72.6, + "samples": 72, + "stdev": 19.789721363367267, + "sum": 1321.6666666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 274.6666666666667, + "max": 565.2, + "mean": 164.5638888888889, + "median": 158.23333333333335, + "min": 3.6, + "non_zero_mean": 164.5638888888889, + "non_zero_median": 158.23333333333335, + "percentile25": 9.633333333333333, + "percentile75": 284.3, + "percentile90": 375.75333333333333, + "percentile99": 552.8460000000001, + "percentile999": 563.9646, + "range": 561.6, + "samples": 72, + "stdev": 163.23388672764253, + "sum": 11848.6 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 72, + "stdev": 0.0, + "sum": 4.8000000000000025 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003703703703703704, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.077333333333335, + "percentile999": 0.24773333333333386, + "range": 0.26666666666666666, + "samples": 72, + "stdev": 0.03142696805273544, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 106.5, + "max": 154.93333333333334, + "mean": 61.794444444444444, + "median": 59.2, + "min": 1.0, + "non_zero_mean": 61.794444444444444, + "non_zero_median": 59.2, + "percentile25": 5.05, + "percentile75": 111.55, + "percentile90": 138.86666666666665, + "percentile99": 154.69666666666666, + "percentile999": 154.90966666666668, + "range": 153.93333333333334, + "samples": 72, + "stdev": 53.69692932647389, + "sum": 4449.2 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "60,64,70,68,67,62,63,61,69,85,76,66,79,72,71,78,75,74,77,82,83,81,80,84,91,90,94,93,87,88,92,97,98,99,101,104,106,108,109,102,103,107,95,105,114,113,115,116,111,110,120,121,123,122,125,118,119,124,127,128,130,132,100,126,133,129,134,131,112,136,137,139,140,141,143,135,117,138,146,150,147,144,149,152,151,156,157,154,160,159,155,163,164,158,165,161,153,162,166,167,171,170,168,174,172,169,187,179,176,177,175,178,182,173,181,183,180,191,189,185,195,184,192,188,190,186,194,193,196,197,199,200,201,198,202,204,206,205,203,207,208,210,211,212,213,215,216,209,217,214,221,220,218,219,224,222,225,223,228,229,230,233,237,234,232,236,227,238,244,242,239,241,235,243,245,240,249,246,250,247,231,248,251,226,253,254,260,255,258,252,261,262,257,263,259,256,265,264,266,267,269,268,270,273,272,274,280,271,275,283,281,286,279,278,284,276,277,282,290,285,298,288,287,292,294,297,295,291,289,301,296,299,305,293,300,309,304,310,307,312,315,314,302,303,308,313,306,311,316,324,317,318,320,322,321,325,327,330,329,323,331,328,326,332,335,339,336,334,337,338,342,346,344,340,347,341,348,345,349,354,351,355,343,359,350,353,319,352,356,333,358,357,360,362,361,363,365,366,364,367,374,368,371,375,369,378,370,383,379,373,372,382,385,380,391,381,377,384,388,376,386,387,392,393,390,394,389,396,395,403,404,406,399,400,412,405,401,398,397,409,407,416,402,414,413,410,417,411,415,419,420,422,421,424,418,425,427,430,428,432,431,429,433,434,426,442,435,423,441,437,443,439,445,444,438,436,447,408,449,440,450,448,452,455,453,458,461,454,457,456,459,464,460,465,446,466,462,467,451,463,468,470,469,474,471,473,472,476,482,477,479,478,475,481,483,480,484,492,493,485,490,488,487,491,495,489,486,502,496,494,506,497,505,504,499,500,498,508,512,514,503,513,509,511,501,507,510,515,518,516,517,519,523,525,522,520,521,532,529,530,531,526,528,537,535,533,527,534,538,543,541,540,542,524,539,544,547,549,552,555,548,553,559,550,557,560,551,556,558,563,545,562,536,546,554,561,565,564,566,567", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-11T22:27:11.593510Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2022-01-11T22:27:16.386845Z 2022-01-11T22:27:16.372041Z 2022-01-11T22:27:16.289499Z 2022-01-11T22:27:15.679011Z 2022-01-11T22:27:15.578183Z 2022-01-11T22:27:15.557023Z 2022-01-11T22:27:15.486348Z 2022-01-11T22:27:15.481073Z 2022-01-11T22:27:15.400623Z 2022-01-11T22:27:15.244336Z 2022-01-11T22:27:15.175307Z 2022-01-11T22:27:14.627491Z 2022-01-11T22:27:14.438118Z 2022-01-11T22:27:14.388070Z 2022-01-11T22:27:14.369847Z 2022-01-11T22:27:14.200563Z 2022-01-11T22:27:13.865616Z 2022-01-11T22:27:13.800971Z 2022-01-11T22:27:13.719540Z 2022-01-11T22:27:13.712512Z 2022-01-11T22:27:13.701204Z 2022-01-11T22:27:13.432708Z 2022-01-11T22:27:13.421966Z 2022-01-11T22:27:13.277529Z 2022-01-11T22:27:13.007744Z 2022-01-11T22:27:12.947199Z 2022-01-11T22:27:12.878906Z 2022-01-11T22:27:12.854934Z 2022-01-11T22:27:12.611603Z 2022-01-11T22:27:12.590912Z 2022-01-11T22:27:12.314359Z 2022-01-11T22:27:12.193202Z 2022-01-11T22:27:12.030558Z 2022-01-11T22:27:11.695711Z 2022-01-11T22:27:11.593510Z 2022-01-11T22:27:32.052066Z 2022-01-11T22:27:31.943502Z 2022-01-11T22:27:29.542754Z 2022-01-11T22:27:28.585369Z 2022-01-11T22:27:27.824858Z 2022-01-11T22:27:27.420783Z 2022-01-11T22:27:27.400545Z 2022-01-11T22:27:27.119597Z 2022-01-11T22:27:27.060289Z 2022-01-11T22:27:26.966345Z 2022-01-11T22:27:26.838134Z 2022-01-11T22:27:26.346952Z 2022-01-11T22:27:26.167099Z 2022-01-11T22:27:25.131961Z 2022-01-11T22:27:25.006024Z 2022-01-11T22:27:24.371190Z 2022-01-11T22:27:24.177411Z 2022-01-11T22:27:23.829354Z 2022-01-11T22:27:23.801480Z 2022-01-11T22:27:23.250072Z 2022-01-11T22:27:22.824263Z 2022-01-11T22:27:22.303385Z 2022-01-11T22:27:21.950891Z 2022-01-11T22:27:21.893475Z 2022-01-11T22:27:21.634343Z 2022-01-11T22:27:21.413829Z 2022-01-11T22:27:21.396319Z 2022-01-11T22:27:21.378746Z 2022-01-11T22:27:21.366034Z 2022-01-11T22:27:21.165128Z 2022-01-11T22:27:20.681769Z 2022-01-11T22:27:20.592911Z 2022-01-11T22:27:20.392276Z 2022-01-11T22:27:20.357735Z 2022-01-11T22:27:20.353852Z 2022-01-11T22:27:20.337931Z 2022-01-11T22:27:20.276517Z 2022-01-11T22:27:20.268624Z 2022-01-11T22:27:20.019473Z 2022-01-11T22:27:19.573586Z 2022-01-11T22:27:19.529549Z 2022-01-11T22:27:19.436932Z 2022-01-11T22:27:19.403225Z 2022-01-11T22:27:19.318586Z 2022-01-11T22:27:19.294280Z 2022-01-11T22:27:19.271796Z 2022-01-11T22:27:19.167205Z 2022-01-11T22:27:18.598336Z 2022-01-11T22:27:18.522289Z 2022-01-11T22:27:18.466596Z 2022-01-11T22:27:18.344671Z 2022-01-11T22:27:18.332110Z 2022-01-11T22:27:18.198468Z 2022-01-11T22:27:18.066596Z 2022-01-11T22:27:17.875610Z 2022-01-11T22:27:17.623573Z 2022-01-11T22:27:17.541612Z 2022-01-11T22:27:17.496669Z 2022-01-11T22:27:17.368980Z 2022-01-11T22:27:17.363768Z 2022-01-11T22:27:17.330932Z 2022-01-11T22:27:17.242195Z 2022-01-11T22:27:17.234019Z 2022-01-11T22:27:16.463564Z 2022-01-11T22:27:16.394034Z 2022-01-11T22:31:38.320639Z 2022-01-11T22:31:38.229523Z 2022-01-11T22:31:38.052630Z 2022-01-11T22:31:37.658049Z 2022-01-11T22:31:35.942397Z 2022-01-11T22:31:35.581201Z 2022-01-11T22:31:34.797599Z 2022-01-11T22:31:34.794994Z 2022-01-11T22:31:34.220004Z 2022-01-11T22:31:34.132762Z 2022-01-11T22:31:34.072994Z 2022-01-11T22:31:34.065244Z 2022-01-11T22:31:33.201082Z 2022-01-11T22:31:33.173050Z 2022-01-11T22:31:32.762498Z 2022-01-11T22:31:32.364736Z 2022-01-11T22:31:31.785247Z 2022-01-11T22:31:31.397624Z 2022-01-11T22:31:31.194777Z 2022-01-11T22:31:31.145387Z 2022-01-11T22:31:31.104798Z 2022-01-11T22:31:30.729715Z 2022-01-11T22:31:29.627043Z 2022-01-11T22:31:29.591178Z 2022-01-11T22:31:29.506220Z 2022-01-11T22:31:29.297213Z 2022-01-11T22:31:29.140454Z 2022-01-11T22:31:28.888445Z 2022-01-11T22:31:28.193447Z 2022-01-11T22:31:28.070474Z 2022-01-11T22:31:27.993562Z 2022-01-11T22:31:27.989396Z 2022-01-11T22:31:27.868270Z 2022-01-11T22:31:27.804149Z 2022-01-11T22:31:27.491295Z 2022-01-11T22:31:27.421938Z 2022-01-11T22:31:27.156739Z 2022-01-11T22:31:27.157112Z 2022-01-11T22:31:27.115082Z 2022-01-11T22:31:26.871761Z 2022-01-11T22:31:26.790578Z 2022-01-11T22:31:26.266759Z 2022-01-11T22:31:26.157618Z 2022-01-11T22:31:25.944128Z 2022-01-11T22:31:25.822738Z 2022-01-11T22:31:25.579226Z 2022-01-11T22:31:25.413706Z 2022-01-11T22:31:25.350084Z 2022-01-11T22:31:25.308434Z 2022-01-11T22:31:24.945399Z 2022-01-11T22:31:24.871999Z 2022-01-11T22:31:24.833259Z 2022-01-11T22:31:24.688951Z 2022-01-11T22:31:24.304930Z 2022-01-11T22:31:24.278887Z 2022-01-11T22:31:24.003464Z 2022-01-11T22:31:23.995971Z 2022-01-11T22:31:23.976850Z 2022-01-11T22:31:23.935315Z 2022-01-11T22:31:23.813815Z 2022-01-11T22:31:23.714519Z 2022-01-11T22:31:23.626139Z 2022-01-11T22:31:23.198402Z 2022-01-11T22:31:22.905996Z 2022-01-11T22:31:22.869068Z 2022-01-11T22:31:22.866300Z 2022-01-11T22:31:22.646194Z 2022-01-11T22:31:22.472586Z 2022-01-11T22:31:22.383341Z 2022-01-11T22:31:22.305370Z 2022-01-11T22:31:22.288891Z 2022-01-11T22:31:22.216945Z 2022-01-11T22:31:22.116489Z 2022-01-11T22:31:22.094269Z 2022-01-11T22:31:21.894228Z 2022-01-11T22:31:21.808345Z 2022-01-11T22:31:21.552019Z 2022-01-11T22:31:21.469114Z 2022-01-11T22:31:21.403517Z 2022-01-11T22:31:21.306651Z 2022-01-11T22:31:21.221261Z 2022-01-11T22:31:21.063651Z 2022-01-11T22:31:20.950099Z 2022-01-11T22:31:20.649820Z 2022-01-11T22:31:20.647745Z 2022-01-11T22:31:20.592970Z 2022-01-11T22:31:20.569371Z 2022-01-11T22:31:20.201708Z 2022-01-11T22:31:20.195643Z 2022-01-11T22:31:20.045092Z 2022-01-11T22:31:20.025901Z 2022-01-11T22:31:19.985733Z 2022-01-11T22:31:19.808974Z 2022-01-11T22:31:19.413274Z 2022-01-11T22:31:19.367183Z 2022-01-11T22:31:19.344486Z 2022-01-11T22:31:19.307058Z 2022-01-11T22:31:19.116506Z 2022-01-11T22:31:19.108439Z 2022-01-11T22:31:18.914094Z 2022-01-11T22:35:26.328896Z 2022-01-11T22:35:26.285695Z 2022-01-11T22:35:26.097872Z 2022-01-11T22:35:25.976872Z 2022-01-11T22:35:25.620621Z 2022-01-11T22:35:25.564860Z 2022-01-11T22:35:25.353508Z 2022-01-11T22:35:25.150279Z 2022-01-11T22:35:25.036592Z 2022-01-11T22:35:24.976501Z 2022-01-11T22:35:24.936305Z 2022-01-11T22:35:24.702931Z 2022-01-11T22:35:24.694022Z 2022-01-11T22:35:24.644405Z 2022-01-11T22:35:24.627683Z 2022-01-11T22:35:24.525610Z 2022-01-11T22:35:24.495613Z 2022-01-11T22:35:24.469328Z 2022-01-11T22:35:23.753471Z 2022-01-11T22:35:23.452660Z 2022-01-11T22:35:23.293591Z 2022-01-11T22:35:23.103473Z 2022-01-11T22:35:23.092344Z 2022-01-11T22:35:23.059096Z 2022-01-11T22:35:23.003728Z 2022-01-11T22:35:22.902924Z 2022-01-11T22:35:22.836712Z 2022-01-11T22:35:22.630546Z 2022-01-11T22:35:22.453850Z 2022-01-11T22:35:22.373218Z 2022-01-11T22:35:22.340161Z 2022-01-11T22:35:22.236596Z 2022-01-11T22:35:22.229385Z 2022-01-11T22:35:22.217292Z 2022-01-11T22:35:22.202718Z 2022-01-11T22:35:22.015676Z 2022-01-11T22:35:21.534410Z 2022-01-11T22:35:21.373087Z 2022-01-11T22:35:21.318653Z 2022-01-11T22:35:21.302270Z 2022-01-11T22:35:21.158485Z 2022-01-11T22:35:21.080835Z 2022-01-11T22:35:20.902448Z 2022-01-11T22:35:20.776244Z 2022-01-11T22:35:20.718972Z 2022-01-11T22:35:20.638685Z 2022-01-11T22:35:20.636797Z 2022-01-11T22:35:20.601014Z 2022-01-11T22:35:20.338876Z 2022-01-11T22:35:20.292847Z 2022-01-11T22:35:19.906033Z 2022-01-11T22:35:19.806717Z 2022-01-11T22:35:19.690795Z 2022-01-11T22:35:19.642644Z 2022-01-11T22:35:19.544762Z 2022-01-11T22:35:19.405109Z 2022-01-11T22:35:19.371202Z 2022-01-11T22:35:19.282639Z 2022-01-11T22:35:19.181196Z 2022-01-11T22:35:19.166599Z 2022-01-11T22:35:19.139778Z 2022-01-11T22:35:19.085277Z 2022-01-11T22:35:18.968605Z 2022-01-11T22:35:18.835772Z 2022-01-11T22:35:18.714790Z 2022-01-11T22:35:18.655982Z 2022-01-11T22:35:18.528837Z 2022-01-11T22:35:18.481759Z 2022-01-11T22:35:18.169713Z 2022-01-11T22:35:18.132369Z 2022-01-11T22:35:18.012445Z 2022-01-11T22:35:17.999070Z 2022-01-11T22:35:17.849773Z 2022-01-11T22:35:17.816561Z 2022-01-11T22:35:17.786429Z 2022-01-11T22:35:17.704839Z 2022-01-11T22:35:17.561461Z 2022-01-11T22:35:17.551459Z 2022-01-11T22:35:17.504597Z 2022-01-11T22:35:17.397696Z 2022-01-11T22:35:17.120457Z 2022-01-11T22:35:17.030348Z 2022-01-11T22:35:16.948269Z 2022-01-11T22:35:16.939193Z 2022-01-11T22:35:16.933413Z 2022-01-11T22:35:16.816372Z 2022-01-11T22:35:16.771893Z 2022-01-11T22:35:16.740192Z 2022-01-11T22:35:16.710077Z 2022-01-11T22:35:16.706042Z 2022-01-11T22:35:16.566996Z 2022-01-11T22:35:16.540283Z 2022-01-11T22:35:16.327580Z 2022-01-11T22:35:16.273365Z 2022-01-11T22:35:16.136796Z 2022-01-11T22:35:16.028987Z 2022-01-11T22:35:15.861630Z 2022-01-11T22:35:15.841828Z 2022-01-11T22:35:15.711873Z 2022-01-11T22:35:15.413685Z 2022-01-11T22:39:24.265843Z 2022-01-11T22:39:23.794392Z 2022-01-11T22:39:23.618937Z 2022-01-11T22:39:23.525275Z 2022-01-11T22:39:23.431873Z 2022-01-11T22:39:23.407601Z 2022-01-11T22:39:23.233471Z 2022-01-11T22:39:23.028834Z 2022-01-11T22:39:22.960751Z 2022-01-11T22:39:22.791607Z 2022-01-11T22:39:22.727730Z 2022-01-11T22:39:22.649186Z 2022-01-11T22:39:22.640457Z 2022-01-11T22:39:22.513610Z 2022-01-11T22:39:22.496727Z 2022-01-11T22:39:22.452285Z 2022-01-11T22:39:22.373738Z 2022-01-11T22:39:22.241446Z 2022-01-11T22:39:21.876286Z 2022-01-11T22:39:21.784426Z 2022-01-11T22:39:21.660665Z 2022-01-11T22:39:21.647790Z 2022-01-11T22:39:21.508273Z 2022-01-11T22:39:21.499697Z 2022-01-11T22:39:21.384953Z 2022-01-11T22:39:21.114333Z 2022-01-11T22:39:21.106468Z 2022-01-11T22:39:21.107556Z 2022-01-11T22:39:20.998618Z 2022-01-11T22:39:20.779223Z 2022-01-11T22:39:20.457585Z 2022-01-11T22:39:20.396288Z 2022-01-11T22:39:20.316528Z 2022-01-11T22:39:20.130652Z 2022-01-11T22:39:20.069976Z 2022-01-11T22:39:19.944078Z 2022-01-11T22:39:19.874014Z 2022-01-11T22:39:19.701676Z 2022-01-11T22:39:19.693654Z 2022-01-11T22:39:19.582352Z 2022-01-11T22:39:19.504439Z 2022-01-11T22:39:19.476900Z 2022-01-11T22:39:19.458361Z 2022-01-11T22:39:19.425526Z 2022-01-11T22:39:19.405851Z 2022-01-11T22:39:19.294339Z 2022-01-11T22:39:19.030368Z 2022-01-11T22:39:18.886180Z 2022-01-11T22:39:18.881224Z 2022-01-11T22:39:18.823870Z 2022-01-11T22:39:18.693825Z 2022-01-11T22:39:18.434479Z 2022-01-11T22:39:18.430970Z 2022-01-11T22:39:18.348523Z 2022-01-11T22:39:18.310750Z 2022-01-11T22:39:18.180511Z 2022-01-11T22:39:18.116467Z 2022-01-11T22:39:17.945800Z 2022-01-11T22:39:17.882533Z 2022-01-11T22:39:17.797870Z 2022-01-11T22:39:17.698946Z 2022-01-11T22:39:17.686231Z 2022-01-11T22:39:17.646684Z 2022-01-11T22:39:17.537252Z 2022-01-11T22:39:17.531501Z 2022-01-11T22:39:17.506290Z 2022-01-11T22:39:17.504107Z 2022-01-11T22:39:17.319006Z 2022-01-11T22:39:17.252969Z 2022-01-11T22:39:17.229802Z 2022-01-11T22:39:17.186349Z 2022-01-11T22:39:17.147581Z 2022-01-11T22:39:17.144427Z 2022-01-11T22:39:17.006866Z 2022-01-11T22:39:16.783790Z 2022-01-11T22:39:16.629397Z 2022-01-11T22:39:16.553403Z 2022-01-11T22:39:16.531496Z 2022-01-11T22:39:16.431814Z 2022-01-11T22:39:16.346945Z 2022-01-11T22:39:16.303871Z 2022-01-11T22:39:16.168242Z 2022-01-11T22:39:16.003303Z 2022-01-11T22:39:15.976717Z 2022-01-11T22:39:15.923535Z 2022-01-11T22:39:15.814368Z 2022-01-11T22:39:15.779249Z 2022-01-11T22:39:15.583337Z 2022-01-11T22:39:15.506241Z 2022-01-11T22:39:15.483546Z 2022-01-11T22:39:15.437216Z 2022-01-11T22:39:15.376067Z 2022-01-11T22:39:15.372068Z 2022-01-11T22:39:15.354295Z 2022-01-11T22:39:15.350512Z 2022-01-11T22:39:15.235680Z 2022-01-11T22:39:15.142766Z 2022-01-11T22:39:15.023923Z 2022-01-11T22:39:14.903094Z 2022-01-11T22:39:14.797788Z 2022-01-11T22:43:31.014925Z 2022-01-11T22:43:30.838201Z 2022-01-11T22:43:30.206446Z 2022-01-11T22:43:29.942003Z 2022-01-11T22:43:28.839042Z 2022-01-11T22:43:28.446705Z 2022-01-11T22:43:28.374417Z 2022-01-11T22:43:28.349914Z 2022-01-11T22:43:27.849934Z 2022-01-11T22:43:27.669120Z 2022-01-11T22:43:27.666247Z 2022-01-11T22:43:27.571782Z 2022-01-11T22:43:27.522848Z 2022-01-11T22:43:27.488539Z 2022-01-11T22:43:27.422473Z 2022-01-11T22:43:27.373571Z 2022-01-11T22:43:27.346081Z 2022-01-11T22:43:27.134208Z 2022-01-11T22:43:27.015180Z 2022-01-11T22:43:26.984613Z 2022-01-11T22:43:26.918158Z 2022-01-11T22:43:26.862724Z 2022-01-11T22:43:26.478087Z 2022-01-11T22:43:25.863337Z 2022-01-11T22:43:25.839835Z 2022-01-11T22:43:25.743561Z 2022-01-11T22:43:25.516552Z 2022-01-11T22:43:25.488554Z 2022-01-11T22:43:25.175006Z 2022-01-11T22:43:24.941300Z 2022-01-11T22:43:24.882724Z 2022-01-11T22:43:24.754006Z 2022-01-11T22:43:24.744471Z 2022-01-11T22:43:24.724537Z 2022-01-11T22:43:24.719961Z 2022-01-11T22:43:24.442594Z 2022-01-11T22:43:24.419493Z 2022-01-11T22:43:24.416643Z 2022-01-11T22:43:24.330416Z 2022-01-11T22:43:24.165557Z 2022-01-11T22:43:23.927330Z 2022-01-11T22:43:23.805632Z 2022-01-11T22:43:23.717298Z 2022-01-11T22:43:23.694759Z 2022-01-11T22:43:23.196322Z 2022-01-11T22:43:23.101012Z 2022-01-11T22:43:22.973489Z 2022-01-11T22:43:22.969233Z 2022-01-11T22:43:22.955078Z 2022-01-11T22:43:22.665878Z 2022-01-11T22:43:22.611878Z 2022-01-11T22:43:22.441248Z 2022-01-11T22:43:21.997173Z 2022-01-11T22:43:21.918689Z 2022-01-11T22:43:21.778424Z 2022-01-11T22:43:21.597724Z 2022-01-11T22:43:21.557259Z 2022-01-11T22:43:21.294764Z 2022-01-11T22:43:21.189007Z 2022-01-11T22:43:20.917302Z 2022-01-11T22:43:20.657961Z 2022-01-11T22:43:20.597141Z 2022-01-11T22:43:20.540617Z 2022-01-11T22:43:20.501566Z 2022-01-11T22:43:20.510048Z 2022-01-11T22:43:20.319408Z 2022-01-11T22:43:20.221985Z 2022-01-11T22:43:20.078192Z 2022-01-11T22:43:20.033184Z 2022-01-11T22:43:20.013787Z 2022-01-11T22:43:20.011638Z 2022-01-11T22:43:19.991223Z 2022-01-11T22:43:19.504385Z 2022-01-11T22:43:19.383330Z 2022-01-11T22:43:19.356909Z 2022-01-11T22:43:19.132456Z 2022-01-11T22:43:19.093381Z 2022-01-11T22:43:19.065034Z 2022-01-11T22:43:19.036648Z 2022-01-11T22:43:19.022751Z 2022-01-11T22:43:18.932044Z 2022-01-11T22:43:18.830690Z 2022-01-11T22:43:18.773295Z 2022-01-11T22:43:18.761462Z 2022-01-11T22:43:18.508342Z 2022-01-11T22:43:18.432405Z 2022-01-11T22:43:18.128321Z 2022-01-11T22:43:18.003722Z 2022-01-11T22:43:17.715249Z 2022-01-11T22:43:17.700424Z 2022-01-11T22:43:17.644238Z 2022-01-11T22:43:17.611900Z 2022-01-11T22:43:17.598290Z 2022-01-11T22:43:17.496156Z 2022-01-11T22:43:16.943195Z 2022-01-11T22:43:16.924875Z 2022-01-11T22:43:16.707834Z 2022-01-11T22:43:16.121968Z 2022-01-11T22:43:15.711414Z 2022-01-11T22:43:15.693066Z", + "created_diff": 15.114045200000001, + "ended": "2022-01-11T22:44:59.974535Z", + "error_job_ids": " []", + "event_first": "2022-01-11T22:28:24.757794Z 2022-01-11T22:28:13.111012Z 2022-01-11T22:28:29.277004Z 2022-01-11T22:28:02.223512Z 2022-01-11T22:28:31.217625Z 2022-01-11T22:28:00.688197Z 2022-01-11T22:28:20.362855Z 2022-01-11T22:27:49.465767Z 2022-01-11T22:28:18.732229Z 2022-01-11T22:28:19.954529Z 2022-01-11T22:27:50.421124Z 2022-01-11T22:27:33.160005Z 2022-01-11T22:27:50.114912Z 2022-01-11T22:27:33.410836Z 2022-01-11T22:27:48.828319Z 2022-01-11T22:27:31.917850Z 2022-01-11T22:27:43.213671Z 2022-01-11T22:27:30.190058Z 2022-01-11T22:27:43.467599Z 2022-01-11T22:28:13.122371Z 2022-01-11T22:27:42.719572Z 2022-01-11T22:29:28.425217Z 2022-01-11T22:27:38.476824Z 2022-01-11T22:27:37.927799Z 2022-01-11T22:27:39.242915Z 2022-01-11T22:28:49.472812Z 2022-01-11T22:27:29.008794Z 2022-01-11T22:29:19.682873Z 2022-01-11T22:27:31.899637Z 2022-01-11T22:29:08.360897Z 2022-01-11T22:27:35.232834Z 2022-01-11T22:28:58.895877Z 2022-01-11T22:27:22.391659Z 2022-01-11T22:27:29.546831Z 2022-01-11T22:27:20.298346Z 2022-01-11T22:28:58.975527Z 2022-01-11T22:28:46.594276Z 2022-01-11T22:29:00.696630Z 2022-01-11T22:28:48.105011Z 2022-01-11T22:28:59.539002Z 2022-01-11T22:28:47.340998Z 2022-01-11T22:28:58.468554Z 2022-01-11T22:28:43.472866Z 2022-01-11T22:29:00.940958Z 2022-01-11T22:28:57.736657Z 2022-01-11T22:28:53.909738Z 2022-01-11T22:28:44.244305Z 2022-01-11T22:28:54.976610Z 2022-01-11T22:28:46.344448Z 2022-01-11T22:28:58.214791Z 2022-01-11T22:28:47.977630Z 2022-01-11T22:28:43.247291Z 2022-01-11T22:28:53.073118Z 2022-01-11T22:28:44.476692Z 2022-01-11T22:28:50.776192Z 2022-01-11T22:28:39.427804Z 2022-01-11T22:28:54.565524Z 2022-01-11T22:28:45.331259Z 2022-01-11T22:28:55.191096Z 2022-01-11T22:28:39.619797Z 2022-01-11T22:28:54.583322Z 2022-01-11T22:28:34.519093Z 2022-01-11T22:28:50.805960Z 2022-01-11T22:28:41.982124Z 2022-01-11T22:28:54.309220Z 2022-01-11T22:28:38.173811Z 2022-01-11T22:28:49.587679Z 2022-01-11T22:28:39.650715Z 2022-01-11T22:28:45.637392Z 2022-01-11T22:28:40.168463Z 2022-01-11T22:28:46.856521Z 2022-01-11T22:28:35.173678Z 2022-01-11T22:28:40.533630Z 2022-01-11T22:28:31.031784Z 2022-01-11T22:28:43.204555Z 2022-01-11T22:28:32.296923Z 2022-01-11T22:28:39.872256Z 2022-01-11T22:28:33.757927Z 2022-01-11T22:28:45.858720Z 2022-01-11T22:28:32.039855Z 2022-01-11T22:28:42.703794Z 2022-01-11T22:28:31.543530Z 2022-01-11T22:28:46.139528Z 2022-01-11T22:28:26.730854Z 2022-01-11T22:28:39.025911Z 2022-01-11T22:28:27.671221Z 2022-01-11T22:28:36.361543Z 2022-01-11T22:28:32.775802Z 2022-01-11T22:28:33.603369Z 2022-01-11T22:28:27.301849Z 2022-01-11T22:28:37.070551Z 2022-01-11T22:28:24.977864Z 2022-01-11T22:28:32.816629Z 2022-01-11T22:28:20.711906Z 2022-01-11T22:28:31.977611Z 2022-01-11T22:28:21.471820Z 2022-01-11T22:28:33.598280Z 2022-01-11T22:28:18.817253Z 2022-01-11T22:28:31.402607Z 2022-01-11T22:28:15.021433Z 2022-01-11T22:32:54.229913Z 2022-01-11T22:32:56.817738Z 2022-01-11T22:32:55.118770Z 2022-01-11T22:32:54.470530Z 2022-01-11T22:32:56.771753Z 2022-01-11T22:32:57.254503Z 2022-01-11T22:32:56.967909Z 2022-01-11T22:32:54.426680Z 2022-01-11T22:32:54.865926Z 2022-01-11T22:32:53.624562Z 2022-01-11T22:32:55.621836Z 2022-01-11T22:32:52.269674Z 2022-01-11T22:32:55.383844Z 2022-01-11T22:32:54.858569Z 2022-01-11T22:32:54.000528Z 2022-01-11T22:32:53.748424Z 2022-01-11T22:32:47.077599Z 2022-01-11T22:32:48.435231Z 2022-01-11T22:32:53.041453Z 2022-01-11T22:32:50.222685Z 2022-01-11T22:32:48.563537Z 2022-01-11T22:32:54.229913Z 2022-01-11T22:32:53.501516Z 2022-01-11T22:32:49.029541Z 2022-01-11T22:32:52.612996Z 2022-01-11T22:32:51.809863Z 2022-01-11T22:32:52.994707Z 2022-01-11T22:32:50.631077Z 2022-01-11T22:32:50.140781Z 2022-01-11T22:32:48.287140Z 2022-01-11T22:32:49.317590Z 2022-01-11T22:32:53.370838Z 2022-01-11T22:32:45.873516Z 2022-01-11T22:32:50.423838Z 2022-01-11T22:32:52.185738Z 2022-01-11T22:32:48.101305Z 2022-01-11T22:32:49.664509Z 2022-01-11T22:32:43.359524Z 2022-01-11T22:32:43.785833Z 2022-01-11T22:32:49.509294Z 2022-01-11T22:32:42.336529Z 2022-01-11T22:32:43.561506Z 2022-01-11T22:32:48.092518Z 2022-01-11T22:32:47.262311Z 2022-01-11T22:32:41.834597Z 2022-01-11T22:32:51.531449Z 2022-01-11T22:32:40.548613Z 2022-01-11T22:32:43.813612Z 2022-01-11T22:32:38.180248Z 2022-01-11T22:32:40.676846Z 2022-01-11T22:32:37.171446Z 2022-01-11T22:32:40.916504Z 2022-01-11T22:32:31.030027Z 2022-01-11T22:32:34.558870Z 2022-01-11T22:32:41.834597Z 2022-01-11T22:32:31.540115Z 2022-01-11T22:32:21.219718Z 2022-01-11T22:32:27.012177Z 2022-01-11T22:32:32.639619Z 2022-01-11T22:32:39.142949Z 2022-01-11T22:32:27.859946Z 2022-01-11T22:32:39.427923Z 2022-01-11T22:32:30.336654Z 2022-01-11T22:32:33.886200Z 2022-01-11T22:32:27.404675Z 2022-01-11T22:32:28.949718Z 2022-01-11T22:32:22.136644Z 2022-01-11T22:32:27.466789Z 2022-01-11T22:32:22.805641Z 2022-01-11T22:32:31.479785Z 2022-01-11T22:32:20.745967Z 2022-01-11T22:32:33.194809Z 2022-01-11T22:32:15.583058Z 2022-01-11T22:32:26.707116Z 2022-01-11T22:32:16.192434Z 2022-01-11T22:32:25.769923Z 2022-01-11T22:31:53.543145Z 2022-01-11T22:31:47.290523Z 2022-01-11T22:31:53.048754Z 2022-01-11T22:31:46.796854Z 2022-01-11T22:31:51.352910Z 2022-01-11T22:31:46.933688Z 2022-01-11T22:31:55.779419Z 2022-01-11T22:31:46.504887Z 2022-01-11T22:31:47.582779Z 2022-01-11T22:31:44.619217Z 2022-01-11T22:31:47.156266Z 2022-01-11T22:31:44.120584Z 2022-01-11T22:31:41.474012Z 2022-01-11T22:31:44.970620Z 2022-01-11T22:31:43.893794Z 2022-01-11T22:31:43.378844Z 2022-01-11T22:31:43.366149Z 2022-01-11T22:31:36.466831Z 2022-01-11T22:31:33.284902Z 2022-01-11T22:31:36.953940Z 2022-01-11T22:31:31.007211Z 2022-01-11T22:31:33.077572Z 2022-01-11T22:31:26.206035Z 2022-01-11T22:31:32.171339Z 2022-01-11T22:36:51.129925Z 2022-01-11T22:36:51.756569Z 2022-01-11T22:36:51.129962Z 2022-01-11T22:36:52.264540Z 2022-01-11T22:36:51.960771Z 2022-01-11T22:36:53.664887Z 2022-01-11T22:36:52.219166Z 2022-01-11T22:36:49.434465Z 2022-01-11T22:36:50.360850Z 2022-01-11T22:36:52.618765Z 2022-01-11T22:36:52.903132Z 2022-01-11T22:36:51.003238Z 2022-01-11T22:36:53.362258Z 2022-01-11T22:36:52.003618Z 2022-01-11T22:36:49.588137Z 2022-01-11T22:36:50.753533Z 2022-01-11T22:36:46.693735Z 2022-01-11T22:36:47.887525Z 2022-01-11T22:36:46.693735Z 2022-01-11T22:36:45.639679Z 2022-01-11T22:36:52.400681Z 2022-01-11T22:36:49.862588Z 2022-01-11T22:36:45.941282Z 2022-01-11T22:36:47.712513Z 2022-01-11T22:36:44.206757Z 2022-01-11T22:36:45.888540Z 2022-01-11T22:36:44.656145Z 2022-01-11T22:36:50.753533Z 2022-01-11T22:36:49.332869Z 2022-01-11T22:36:50.053408Z 2022-01-11T22:36:39.638512Z 2022-01-11T22:36:46.366011Z 2022-01-11T22:36:39.779214Z 2022-01-11T22:36:48.979507Z 2022-01-11T22:36:45.419755Z 2022-01-11T22:36:46.899609Z 2022-01-11T22:36:44.905816Z 2022-01-11T22:36:44.415401Z 2022-01-11T22:36:43.672936Z 2022-01-11T22:36:45.125672Z 2022-01-11T22:36:43.642809Z 2022-01-11T22:36:45.165637Z 2022-01-11T22:36:36.318780Z 2022-01-11T22:36:24.087030Z 2022-01-11T22:36:27.738529Z 2022-01-11T22:36:27.198696Z 2022-01-11T22:36:38.333738Z 2022-01-11T22:36:32.609694Z 2022-01-11T22:36:43.725279Z 2022-01-11T22:36:33.510760Z 2022-01-11T22:36:28.825922Z 2022-01-11T22:36:24.951049Z 2022-01-11T22:36:18.963550Z 2022-01-11T22:36:30.251271Z 2022-01-11T22:36:25.291045Z 2022-01-11T22:36:21.573878Z 2022-01-11T22:36:19.786795Z 2022-01-11T22:36:26.182826Z 2022-01-11T22:36:29.280956Z 2022-01-11T22:36:25.298559Z 2022-01-11T22:36:22.024206Z 2022-01-11T22:36:23.247679Z 2022-01-11T22:36:10.676124Z 2022-01-11T22:36:26.731610Z 2022-01-11T22:36:20.800301Z 2022-01-11T22:36:22.993720Z 2022-01-11T22:36:10.446452Z 2022-01-11T22:36:11.332558Z 2022-01-11T22:36:00.195812Z 2022-01-11T22:36:11.950282Z 2022-01-11T22:36:03.919851Z 2022-01-11T22:36:07.489586Z 2022-01-11T22:36:05.454797Z 2022-01-11T22:36:06.781734Z 2022-01-11T22:36:04.344707Z 2022-01-11T22:36:09.362410Z 2022-01-11T22:36:00.615804Z 2022-01-11T22:36:10.786937Z 2022-01-11T22:36:02.887768Z 2022-01-11T22:36:00.636681Z 2022-01-11T22:35:57.294287Z 2022-01-11T22:35:59.649705Z 2022-01-11T22:35:53.359981Z 2022-01-11T22:36:08.459536Z 2022-01-11T22:35:50.694883Z 2022-01-11T22:35:55.654743Z 2022-01-11T22:35:48.351380Z 2022-01-11T22:35:56.422734Z 2022-01-11T22:35:46.327133Z 2022-01-11T22:35:53.004801Z 2022-01-11T22:35:44.772898Z 2022-01-11T22:35:53.509025Z 2022-01-11T22:35:30.014572Z 2022-01-11T22:35:33.051885Z 2022-01-11T22:35:29.115893Z 2022-01-11T22:35:32.295659Z 2022-01-11T22:35:28.468034Z 2022-01-11T22:35:29.313973Z 2022-01-11T22:35:25.467606Z 2022-01-11T22:35:24.314545Z 2022-01-11T22:40:48.310898Z 2022-01-11T22:40:55.902578Z 2022-01-11T22:40:51.391282Z 2022-01-11T22:40:50.149898Z 2022-01-11T22:40:46.087153Z 2022-01-11T22:40:55.655622Z 2022-01-11T22:40:49.953346Z 2022-01-11T22:40:52.506102Z 2022-01-11T22:40:43.321891Z 2022-01-11T22:40:48.216902Z 2022-01-11T22:40:46.785237Z 2022-01-11T22:40:48.978418Z 2022-01-11T22:40:47.007068Z 2022-01-11T22:40:51.242136Z 2022-01-11T22:40:47.959565Z 2022-01-11T22:40:55.280125Z 2022-01-11T22:40:46.763781Z 2022-01-11T22:40:50.737580Z 2022-01-11T22:40:46.763781Z 2022-01-11T22:40:52.228998Z 2022-01-11T22:40:49.170556Z 2022-01-11T22:40:50.983661Z 2022-01-11T22:40:46.592343Z 2022-01-11T22:40:51.985097Z 2022-01-11T22:40:46.516584Z 2022-01-11T22:40:48.977544Z 2022-01-11T22:40:39.989127Z 2022-01-11T22:40:43.569986Z 2022-01-11T22:40:42.819035Z 2022-01-11T22:40:45.585670Z 2022-01-11T22:40:41.315829Z 2022-01-11T22:40:48.918709Z 2022-01-11T22:40:43.069812Z 2022-01-11T22:40:49.194111Z 2022-01-11T22:40:45.131677Z 2022-01-11T22:40:46.531816Z 2022-01-11T22:40:42.817679Z 2022-01-11T22:40:47.135535Z 2022-01-11T22:40:37.614051Z 2022-01-11T22:40:50.437583Z 2022-01-11T22:40:34.863835Z 2022-01-11T22:40:45.768703Z 2022-01-11T22:40:32.370616Z 2022-01-11T22:40:43.741346Z 2022-01-11T22:40:40.637017Z 2022-01-11T22:40:39.921627Z 2022-01-11T22:40:34.073877Z 2022-01-11T22:40:42.266030Z 2022-01-11T22:40:33.663044Z 2022-01-11T22:40:37.388511Z 2022-01-11T22:40:35.876381Z 2022-01-11T22:40:37.640511Z 2022-01-11T22:40:36.566358Z 2022-01-11T22:40:36.280639Z 2022-01-11T22:40:37.610350Z 2022-01-11T22:40:35.077848Z 2022-01-11T22:40:26.604209Z 2022-01-11T22:40:30.777693Z 2022-01-11T22:40:27.298953Z 2022-01-11T22:40:34.250664Z 2022-01-11T22:40:26.351709Z 2022-01-11T22:40:31.495663Z 2022-01-11T22:40:21.157612Z 2022-01-11T22:40:35.429791Z 2022-01-11T22:40:27.137839Z 2022-01-11T22:40:30.364823Z 2022-01-11T22:40:13.599580Z 2022-01-11T22:40:27.770920Z 2022-01-11T22:40:19.067274Z 2022-01-11T22:40:27.195246Z 2022-01-11T22:40:14.433938Z 2022-01-11T22:40:17.422118Z 2022-01-11T22:40:12.910210Z 2022-01-11T22:40:23.799618Z 2022-01-11T22:40:05.140610Z 2022-01-11T22:39:46.369030Z 2022-01-11T22:40:20.263523Z 2022-01-11T22:39:52.868276Z 2022-01-11T22:39:42.135156Z 2022-01-11T22:39:44.960222Z 2022-01-11T22:39:43.668464Z 2022-01-11T22:39:44.444430Z 2022-01-11T22:39:40.624762Z 2022-01-11T22:39:43.609596Z 2022-01-11T22:39:41.619865Z 2022-01-11T22:39:43.883808Z 2022-01-11T22:39:40.854009Z 2022-01-11T22:39:42.402564Z 2022-01-11T22:39:38.839687Z 2022-01-11T22:39:42.603925Z 2022-01-11T22:39:38.840466Z 2022-01-11T22:39:39.110826Z 2022-01-11T22:39:34.801671Z 2022-01-11T22:39:38.860655Z 2022-01-11T22:39:34.554841Z 2022-01-11T22:39:37.904407Z 2022-01-11T22:39:31.720252Z 2022-01-11T22:39:27.489665Z 2022-01-11T22:39:22.845209Z 2022-01-11T22:39:27.013626Z 2022-01-11T22:44:54.643811Z 2022-01-11T22:44:57.197360Z 2022-01-11T22:44:57.928023Z 2022-01-11T22:44:55.558535Z 2022-01-11T22:44:56.336354Z 2022-01-11T22:44:54.022342Z 2022-01-11T22:44:55.252699Z 2022-01-11T22:44:52.208643Z 2022-01-11T22:44:54.643811Z 2022-01-11T22:44:53.989134Z 2022-01-11T22:44:56.170712Z 2022-01-11T22:44:55.306626Z 2022-01-11T22:44:54.924085Z 2022-01-11T22:44:55.813877Z 2022-01-11T22:44:54.439341Z 2022-01-11T22:44:55.070742Z 2022-01-11T22:44:55.344836Z 2022-01-11T22:44:55.813877Z 2022-01-11T22:44:52.495930Z 2022-01-11T22:44:54.486098Z 2022-01-11T22:44:51.623130Z 2022-01-11T22:44:52.740676Z 2022-01-11T22:44:53.756625Z 2022-01-11T22:44:51.201835Z 2022-01-11T22:44:45.006770Z 2022-01-11T22:44:47.576267Z 2022-01-11T22:44:48.290592Z 2022-01-11T22:44:45.202724Z 2022-01-11T22:44:45.839951Z 2022-01-11T22:44:47.826944Z 2022-01-11T22:44:43.309169Z 2022-01-11T22:44:53.430715Z 2022-01-11T22:44:41.098229Z 2022-01-11T22:44:42.300646Z 2022-01-11T22:44:43.712594Z 2022-01-11T22:44:48.625074Z 2022-01-11T22:44:40.268540Z 2022-01-11T22:44:39.105498Z 2022-01-11T22:44:42.087273Z 2022-01-11T22:44:43.080676Z 2022-01-11T22:44:47.567685Z 2022-01-11T22:44:40.573503Z 2022-01-11T22:44:41.341845Z 2022-01-11T22:44:38.053950Z 2022-01-11T22:44:31.414108Z 2022-01-11T22:44:43.531532Z 2022-01-11T22:44:42.765073Z 2022-01-11T22:44:32.879556Z 2022-01-11T22:44:36.019289Z 2022-01-11T22:44:41.009521Z 2022-01-11T22:44:39.578731Z 2022-01-11T22:44:33.519565Z 2022-01-11T22:44:37.375772Z 2022-01-11T22:44:35.835381Z 2022-01-11T22:44:37.375772Z 2022-01-11T22:44:41.546931Z 2022-01-11T22:44:35.642877Z 2022-01-11T22:44:40.842356Z 2022-01-11T22:44:32.049782Z 2022-01-11T22:44:30.741924Z 2022-01-11T22:44:35.081263Z 2022-01-11T22:44:29.899542Z 2022-01-11T22:44:23.617474Z 2022-01-11T22:44:28.664922Z 2022-01-11T22:44:31.722517Z 2022-01-11T22:44:28.315902Z 2022-01-11T22:44:30.040944Z 2022-01-11T22:44:26.190757Z 2022-01-11T22:44:28.240820Z 2022-01-11T22:44:29.899542Z 2022-01-11T22:44:16.494756Z 2022-01-11T22:44:25.096964Z 2022-01-11T22:44:23.617793Z 2022-01-11T22:44:07.686802Z 2022-01-11T22:44:28.303693Z 2022-01-11T22:44:20.110999Z 2022-01-11T22:44:17.763949Z 2022-01-11T22:44:13.097624Z 2022-01-11T22:44:19.366032Z 2022-01-11T22:44:15.114131Z 2022-01-11T22:44:17.016916Z 2022-01-11T22:44:00.217688Z 2022-01-11T22:44:02.451414Z 2022-01-11T22:43:56.589165Z 2022-01-11T22:44:02.725760Z 2022-01-11T22:43:59.968674Z 2022-01-11T22:44:02.701535Z 2022-01-11T22:43:59.991719Z 2022-01-11T22:43:37.394358Z 2022-01-11T22:43:34.711987Z 2022-01-11T22:43:37.145094Z 2022-01-11T22:43:34.948827Z 2022-01-11T22:43:35.033393Z 2022-01-11T22:43:33.697296Z 2022-01-11T22:43:32.924121Z 2022-01-11T22:43:30.384397Z 2022-01-11T22:43:26.541005Z 2022-01-11T22:43:29.296800Z 2022-01-11T22:43:23.624645Z 2022-01-11T22:43:23.353871Z", + "event_last": "2022-01-11T22:28:28.914388Z 2022-01-11T22:28:17.042568Z 2022-01-11T22:28:33.884354Z 2022-01-11T22:28:05.263540Z 2022-01-11T22:28:37.841671Z 2022-01-11T22:28:04.967830Z 2022-01-11T22:28:24.043447Z 2022-01-11T22:27:51.497752Z 2022-01-11T22:28:22.682486Z 2022-01-11T22:28:25.113385Z 2022-01-11T22:27:53.514378Z 2022-01-11T22:27:35.266837Z 2022-01-11T22:27:53.373562Z 2022-01-11T22:27:35.870648Z 2022-01-11T22:27:51.990600Z 2022-01-11T22:27:32.559238Z 2022-01-11T22:27:45.306429Z 2022-01-11T22:27:31.098821Z 2022-01-11T22:27:45.578446Z 2022-01-11T22:28:16.731621Z 2022-01-11T22:27:44.986956Z 2022-01-11T22:29:28.832674Z 2022-01-11T22:27:39.563507Z 2022-01-11T22:27:38.347572Z 2022-01-11T22:27:40.401883Z 2022-01-11T22:28:49.911598Z 2022-01-11T22:27:30.742208Z 2022-01-11T22:29:20.032829Z 2022-01-11T22:27:33.216376Z 2022-01-11T22:29:08.756704Z 2022-01-11T22:27:36.420494Z 2022-01-11T22:28:59.539334Z 2022-01-11T22:27:23.788571Z 2022-01-11T22:27:30.211294Z 2022-01-11T22:27:22.651762Z 2022-01-11T22:29:00.092825Z 2022-01-11T22:28:47.898829Z 2022-01-11T22:29:01.134760Z 2022-01-11T22:28:49.480969Z 2022-01-11T22:29:00.781490Z 2022-01-11T22:28:48.322543Z 2022-01-11T22:28:59.787718Z 2022-01-11T22:28:45.868584Z 2022-01-11T22:29:03.400503Z 2022-01-11T22:28:59.630000Z 2022-01-11T22:28:56.225367Z 2022-01-11T22:28:46.409469Z 2022-01-11T22:28:57.435362Z 2022-01-11T22:28:47.882494Z 2022-01-11T22:28:59.771677Z 2022-01-11T22:28:50.883494Z 2022-01-11T22:28:45.331259Z 2022-01-11T22:28:55.217135Z 2022-01-11T22:28:46.491465Z 2022-01-11T22:28:53.368307Z 2022-01-11T22:28:41.577934Z 2022-01-11T22:28:56.811168Z 2022-01-11T22:28:47.259029Z 2022-01-11T22:28:57.261371Z 2022-01-11T22:28:41.957623Z 2022-01-11T22:28:56.888354Z 2022-01-11T22:28:38.667623Z 2022-01-11T22:28:53.682820Z 2022-01-11T22:28:44.338582Z 2022-01-11T22:28:57.150399Z 2022-01-11T22:28:40.497580Z 2022-01-11T22:28:51.558380Z 2022-01-11T22:28:42.071634Z 2022-01-11T22:28:49.276633Z 2022-01-11T22:28:43.349689Z 2022-01-11T22:28:50.394589Z 2022-01-11T22:28:38.667623Z 2022-01-11T22:28:44.568492Z 2022-01-11T22:28:34.376657Z 2022-01-11T22:28:47.518561Z 2022-01-11T22:28:35.518963Z 2022-01-11T22:28:43.066403Z 2022-01-11T22:28:37.527571Z 2022-01-11T22:28:49.582364Z 2022-01-11T22:28:35.941534Z 2022-01-11T22:28:45.702850Z 2022-01-11T22:28:35.683706Z 2022-01-11T22:28:49.861013Z 2022-01-11T22:28:29.709516Z 2022-01-11T22:28:42.656473Z 2022-01-11T22:28:31.093815Z 2022-01-11T22:28:40.142386Z 2022-01-11T22:28:35.683706Z 2022-01-11T22:28:38.129421Z 2022-01-11T22:28:30.093568Z 2022-01-11T22:28:41.898957Z 2022-01-11T22:28:27.774582Z 2022-01-11T22:28:36.182457Z 2022-01-11T22:28:24.677545Z 2022-01-11T22:28:37.315330Z 2022-01-11T22:28:25.305567Z 2022-01-11T22:28:38.367725Z 2022-01-11T22:28:22.351490Z 2022-01-11T22:28:35.913174Z 2022-01-11T22:28:18.294908Z 2022-01-11T22:32:56.396526Z 2022-01-11T22:32:58.073329Z 2022-01-11T22:32:56.780142Z 2022-01-11T22:32:56.815094Z 2022-01-11T22:32:58.347112Z 2022-01-11T22:32:58.310373Z 2022-01-11T22:32:58.273883Z 2022-01-11T22:32:56.381852Z 2022-01-11T22:32:56.952611Z 2022-01-11T22:32:56.625829Z 2022-01-11T22:32:57.891221Z 2022-01-11T22:32:55.155739Z 2022-01-11T22:32:57.106996Z 2022-01-11T22:32:56.645720Z 2022-01-11T22:32:55.995424Z 2022-01-11T22:32:56.292184Z 2022-01-11T22:32:50.502356Z 2022-01-11T22:32:52.557535Z 2022-01-11T22:32:55.462351Z 2022-01-11T22:32:53.868506Z 2022-01-11T22:32:51.356563Z 2022-01-11T22:32:56.601084Z 2022-01-11T22:32:55.810076Z 2022-01-11T22:32:52.610001Z 2022-01-11T22:32:54.473269Z 2022-01-11T22:32:54.580648Z 2022-01-11T22:32:56.238547Z 2022-01-11T22:32:53.772636Z 2022-01-11T22:32:52.634788Z 2022-01-11T22:32:51.951535Z 2022-01-11T22:32:51.904485Z 2022-01-11T22:32:56.000702Z 2022-01-11T22:32:48.712949Z 2022-01-11T22:32:53.496205Z 2022-01-11T22:32:54.825539Z 2022-01-11T22:32:51.124524Z 2022-01-11T22:32:52.888566Z 2022-01-11T22:32:45.873516Z 2022-01-11T22:32:47.552403Z 2022-01-11T22:32:52.758598Z 2022-01-11T22:32:44.803988Z 2022-01-11T22:32:46.320560Z 2022-01-11T22:32:50.861070Z 2022-01-11T22:32:50.453544Z 2022-01-11T22:32:45.639639Z 2022-01-11T22:32:54.597034Z 2022-01-11T22:32:43.946356Z 2022-01-11T22:32:47.494576Z 2022-01-11T22:32:41.084405Z 2022-01-11T22:32:43.579634Z 2022-01-11T22:32:39.993372Z 2022-01-11T22:32:44.708912Z 2022-01-11T22:32:35.347336Z 2022-01-11T22:32:37.839638Z 2022-01-11T22:32:45.405376Z 2022-01-11T22:32:35.051297Z 2022-01-11T22:32:25.245494Z 2022-01-11T22:32:31.300849Z 2022-01-11T22:32:36.703537Z 2022-01-11T22:32:43.042904Z 2022-01-11T22:32:30.655381Z 2022-01-11T22:32:42.645015Z 2022-01-11T22:32:35.022791Z 2022-01-11T22:32:37.987843Z 2022-01-11T22:32:30.954509Z 2022-01-11T22:32:32.167864Z 2022-01-11T22:32:26.426369Z 2022-01-11T22:32:31.766572Z 2022-01-11T22:32:27.370384Z 2022-01-11T22:32:35.106862Z 2022-01-11T22:32:24.049353Z 2022-01-11T22:32:36.467650Z 2022-01-11T22:32:20.429422Z 2022-01-11T22:32:31.198524Z 2022-01-11T22:32:20.504968Z 2022-01-11T22:32:29.189556Z 2022-01-11T22:31:56.464222Z 2022-01-11T22:31:51.359117Z 2022-01-11T22:31:55.823183Z 2022-01-11T22:31:48.412808Z 2022-01-11T22:31:53.547904Z 2022-01-11T22:31:48.806249Z 2022-01-11T22:31:58.950391Z 2022-01-11T22:31:48.371428Z 2022-01-11T22:31:49.185584Z 2022-01-11T22:31:46.190850Z 2022-01-11T22:31:49.052584Z 2022-01-11T22:31:45.103378Z 2022-01-11T22:31:42.410821Z 2022-01-11T22:31:46.514354Z 2022-01-11T22:31:45.363231Z 2022-01-11T22:31:44.397417Z 2022-01-11T22:31:44.688594Z 2022-01-11T22:31:37.659782Z 2022-01-11T22:31:34.736411Z 2022-01-11T22:31:38.308236Z 2022-01-11T22:31:32.407132Z 2022-01-11T22:31:34.697007Z 2022-01-11T22:31:27.233573Z 2022-01-11T22:31:34.115639Z 2022-01-11T22:36:52.958641Z 2022-01-11T22:36:54.111827Z 2022-01-11T22:36:52.747223Z 2022-01-11T22:36:53.923784Z 2022-01-11T22:36:53.546042Z 2022-01-11T22:36:54.684444Z 2022-01-11T22:36:53.314181Z 2022-01-11T22:36:51.693400Z 2022-01-11T22:36:52.066477Z 2022-01-11T22:36:54.397462Z 2022-01-11T22:36:54.538756Z 2022-01-11T22:36:53.140371Z 2022-01-11T22:36:54.679519Z 2022-01-11T22:36:54.111827Z 2022-01-11T22:36:51.161806Z 2022-01-11T22:36:52.796990Z 2022-01-11T22:36:49.544604Z 2022-01-11T22:36:49.964412Z 2022-01-11T22:36:49.789672Z 2022-01-11T22:36:48.247136Z 2022-01-11T22:36:54.083631Z 2022-01-11T22:36:52.492363Z 2022-01-11T22:36:48.904074Z 2022-01-11T22:36:50.081756Z 2022-01-11T22:36:47.018205Z 2022-01-11T22:36:48.513357Z 2022-01-11T22:36:47.922748Z 2022-01-11T22:36:53.047477Z 2022-01-11T22:36:51.736101Z 2022-01-11T22:36:53.047477Z 2022-01-11T22:36:41.354505Z 2022-01-11T22:36:48.930480Z 2022-01-11T22:36:42.983522Z 2022-01-11T22:36:51.537630Z 2022-01-11T22:36:48.119566Z 2022-01-11T22:36:50.140432Z 2022-01-11T22:36:47.883584Z 2022-01-11T22:36:46.675193Z 2022-01-11T22:36:45.725577Z 2022-01-11T22:36:47.276895Z 2022-01-11T22:36:46.586635Z 2022-01-11T22:36:48.511916Z 2022-01-11T22:36:39.324312Z 2022-01-11T22:36:26.993814Z 2022-01-11T22:36:31.569559Z 2022-01-11T22:36:30.578761Z 2022-01-11T22:36:40.827418Z 2022-01-11T22:36:34.560071Z 2022-01-11T22:36:46.666921Z 2022-01-11T22:36:36.637407Z 2022-01-11T22:36:31.792538Z 2022-01-11T22:36:28.519496Z 2022-01-11T22:36:22.618580Z 2022-01-11T22:36:33.486900Z 2022-01-11T22:36:28.341522Z 2022-01-11T22:36:25.363886Z 2022-01-11T22:36:22.876535Z 2022-01-11T22:36:30.443435Z 2022-01-11T22:36:32.689558Z 2022-01-11T22:36:29.986404Z 2022-01-11T22:36:25.607521Z 2022-01-11T22:36:26.954682Z 2022-01-11T22:36:14.678705Z 2022-01-11T22:36:30.305350Z 2022-01-11T22:36:24.404308Z 2022-01-11T22:36:27.029468Z 2022-01-11T22:36:15.706214Z 2022-01-11T22:36:16.429949Z 2022-01-11T22:36:03.267594Z 2022-01-11T22:36:17.835385Z 2022-01-11T22:36:08.229020Z 2022-01-11T22:36:12.954463Z 2022-01-11T22:36:10.473531Z 2022-01-11T22:36:11.028712Z 2022-01-11T22:36:09.112196Z 2022-01-11T22:36:15.485603Z 2022-01-11T22:36:04.892706Z 2022-01-11T22:36:17.257303Z 2022-01-11T22:36:07.340575Z 2022-01-11T22:36:03.254207Z 2022-01-11T22:36:00.493635Z 2022-01-11T22:36:03.139796Z 2022-01-11T22:35:55.076761Z 2022-01-11T22:36:13.770501Z 2022-01-11T22:35:53.024997Z 2022-01-11T22:35:57.696657Z 2022-01-11T22:35:50.354992Z 2022-01-11T22:36:00.340356Z 2022-01-11T22:35:48.666913Z 2022-01-11T22:35:55.697939Z 2022-01-11T22:35:46.886009Z 2022-01-11T22:35:56.539381Z 2022-01-11T22:35:32.015522Z 2022-01-11T22:35:34.729922Z 2022-01-11T22:35:29.523994Z 2022-01-11T22:35:33.820432Z 2022-01-11T22:35:29.115893Z 2022-01-11T22:35:31.636119Z 2022-01-11T22:35:26.329617Z 2022-01-11T22:35:27.085411Z 2022-01-11T22:40:50.819761Z 2022-01-11T22:40:56.234421Z 2022-01-11T22:40:53.226539Z 2022-01-11T22:40:53.109454Z 2022-01-11T22:40:48.532028Z 2022-01-11T22:40:56.361401Z 2022-01-11T22:40:51.752116Z 2022-01-11T22:40:54.824622Z 2022-01-11T22:40:46.546604Z 2022-01-11T22:40:51.463263Z 2022-01-11T22:40:49.779502Z 2022-01-11T22:40:52.303390Z 2022-01-11T22:40:49.951624Z 2022-01-11T22:40:53.722773Z 2022-01-11T22:40:50.581012Z 2022-01-11T22:40:56.211022Z 2022-01-11T22:40:49.889990Z 2022-01-11T22:40:53.023272Z 2022-01-11T22:40:49.703822Z 2022-01-11T22:40:54.300188Z 2022-01-11T22:40:51.700979Z 2022-01-11T22:40:53.593486Z 2022-01-11T22:40:48.696818Z 2022-01-11T22:40:54.464511Z 2022-01-11T22:40:49.423182Z 2022-01-11T22:40:51.873367Z 2022-01-11T22:40:43.757366Z 2022-01-11T22:40:47.097762Z 2022-01-11T22:40:46.090643Z 2022-01-11T22:40:49.061495Z 2022-01-11T22:40:44.388599Z 2022-01-11T22:40:51.915743Z 2022-01-11T22:40:45.876523Z 2022-01-11T22:40:51.918002Z 2022-01-11T22:40:48.307492Z 2022-01-11T22:40:50.288617Z 2022-01-11T22:40:45.007883Z 2022-01-11T22:40:50.149898Z 2022-01-11T22:40:41.912021Z 2022-01-11T22:40:53.156432Z 2022-01-11T22:40:40.210945Z 2022-01-11T22:40:48.212866Z 2022-01-11T22:40:35.159856Z 2022-01-11T22:40:47.182086Z 2022-01-11T22:40:43.956951Z 2022-01-11T22:40:43.955535Z 2022-01-11T22:40:37.947477Z 2022-01-11T22:40:45.384493Z 2022-01-11T22:40:37.237506Z 2022-01-11T22:40:41.532675Z 2022-01-11T22:40:39.245562Z 2022-01-11T22:40:41.571334Z 2022-01-11T22:40:40.486549Z 2022-01-11T22:40:38.891799Z 2022-01-11T22:40:41.301524Z 2022-01-11T22:40:38.464474Z 2022-01-11T22:40:30.042525Z 2022-01-11T22:40:35.638350Z 2022-01-11T22:40:31.243402Z 2022-01-11T22:40:38.224427Z 2022-01-11T22:40:29.950115Z 2022-01-11T22:40:34.414747Z 2022-01-11T22:40:24.958529Z 2022-01-11T22:40:37.736539Z 2022-01-11T22:40:31.179805Z 2022-01-11T22:40:34.842405Z 2022-01-11T22:40:16.848523Z 2022-01-11T22:40:31.285986Z 2022-01-11T22:40:23.821557Z 2022-01-11T22:40:29.728039Z 2022-01-11T22:40:16.684287Z 2022-01-11T22:40:20.683366Z 2022-01-11T22:40:16.299509Z 2022-01-11T22:40:28.259391Z 2022-01-11T22:40:10.218593Z 2022-01-11T22:39:48.506425Z 2022-01-11T22:40:24.866360Z 2022-01-11T22:39:56.457116Z 2022-01-11T22:39:43.874746Z 2022-01-11T22:39:47.284704Z 2022-01-11T22:39:45.028858Z 2022-01-11T22:39:46.679388Z 2022-01-11T22:39:42.727258Z 2022-01-11T22:39:46.071203Z 2022-01-11T22:39:43.235026Z 2022-01-11T22:39:46.704876Z 2022-01-11T22:39:42.514267Z 2022-01-11T22:39:44.294375Z 2022-01-11T22:39:40.203580Z 2022-01-11T22:39:44.683838Z 2022-01-11T22:39:40.479700Z 2022-01-11T22:39:41.195362Z 2022-01-11T22:39:36.250567Z 2022-01-11T22:39:41.195362Z 2022-01-11T22:39:35.638523Z 2022-01-11T22:39:39.467370Z 2022-01-11T22:39:32.967912Z 2022-01-11T22:39:29.830094Z 2022-01-11T22:39:23.581114Z 2022-01-11T22:39:28.243402Z 2022-01-11T22:44:56.926182Z 2022-01-11T22:44:57.955214Z 2022-01-11T22:44:59.974535Z 2022-01-11T22:44:57.110999Z 2022-01-11T22:44:58.027543Z 2022-01-11T22:44:56.372752Z 2022-01-11T22:44:57.437520Z 2022-01-11T22:44:54.085877Z 2022-01-11T22:44:56.729062Z 2022-01-11T22:44:55.574736Z 2022-01-11T22:44:58.260691Z 2022-01-11T22:44:57.071391Z 2022-01-11T22:44:56.750696Z 2022-01-11T22:44:57.324327Z 2022-01-11T22:44:56.431786Z 2022-01-11T22:44:56.697234Z 2022-01-11T22:44:57.160863Z 2022-01-11T22:44:57.473082Z 2022-01-11T22:44:54.352972Z 2022-01-11T22:44:56.341739Z 2022-01-11T22:44:53.818408Z 2022-01-11T22:44:54.742030Z 2022-01-11T22:44:55.452884Z 2022-01-11T22:44:53.564735Z 2022-01-11T22:44:49.054735Z 2022-01-11T22:44:50.903503Z 2022-01-11T22:44:50.671500Z 2022-01-11T22:44:48.172775Z 2022-01-11T22:44:48.861429Z 2022-01-11T22:44:50.403846Z 2022-01-11T22:44:46.389031Z 2022-01-11T22:44:55.345598Z 2022-01-11T22:44:44.577543Z 2022-01-11T22:44:45.414119Z 2022-01-11T22:44:47.125499Z 2022-01-11T22:44:51.197531Z 2022-01-11T22:44:44.036771Z 2022-01-11T22:44:43.453334Z 2022-01-11T22:44:45.056105Z 2022-01-11T22:44:46.774368Z 2022-01-11T22:44:49.975565Z 2022-01-11T22:44:44.457392Z 2022-01-11T22:44:45.280683Z 2022-01-11T22:44:41.946363Z 2022-01-11T22:44:35.654650Z 2022-01-11T22:44:48.217529Z 2022-01-11T22:44:44.787584Z 2022-01-11T22:44:36.848631Z 2022-01-11T22:44:39.448530Z 2022-01-11T22:44:43.067734Z 2022-01-11T22:44:43.809543Z 2022-01-11T22:44:38.269890Z 2022-01-11T22:44:41.701008Z 2022-01-11T22:44:40.309957Z 2022-01-11T22:44:42.341643Z 2022-01-11T22:44:44.070418Z 2022-01-11T22:44:39.493114Z 2022-01-11T22:44:44.356426Z 2022-01-11T22:44:35.740773Z 2022-01-11T22:44:34.591351Z 2022-01-11T22:44:38.741567Z 2022-01-11T22:44:34.389356Z 2022-01-11T22:44:27.519581Z 2022-01-11T22:44:32.918242Z 2022-01-11T22:44:35.704819Z 2022-01-11T22:44:32.291044Z 2022-01-11T22:44:34.147782Z 2022-01-11T22:44:29.759353Z 2022-01-11T22:44:32.798089Z 2022-01-11T22:44:34.056554Z 2022-01-11T22:44:20.429028Z 2022-01-11T22:44:28.977387Z 2022-01-11T22:44:26.982504Z 2022-01-11T22:44:14.300923Z 2022-01-11T22:44:32.621834Z 2022-01-11T22:44:24.652350Z 2022-01-11T22:44:21.517626Z 2022-01-11T22:44:17.260180Z 2022-01-11T22:44:24.514517Z 2022-01-11T22:44:19.868395Z 2022-01-11T22:44:21.848464Z 2022-01-11T22:44:04.446505Z 2022-01-11T22:44:07.475457Z 2022-01-11T22:43:59.520895Z 2022-01-11T22:44:07.492774Z 2022-01-11T22:44:03.697338Z 2022-01-11T22:44:06.331687Z 2022-01-11T22:44:04.504057Z 2022-01-11T22:43:38.524596Z 2022-01-11T22:43:35.546073Z 2022-01-11T22:43:38.233116Z 2022-01-11T22:43:37.201813Z 2022-01-11T22:43:35.719949Z 2022-01-11T22:43:34.719385Z 2022-01-11T22:43:33.615690Z 2022-01-11T22:43:31.077312Z 2022-01-11T22:43:27.424737Z 2022-01-11T22:43:30.385578Z 2022-01-11T22:43:24.396374Z 2022-01-11T22:43:24.749462Z", + "failed_job_ids": " []", + "finished": "2022-01-11T22:28:35.056252Z 2022-01-11T22:28:22.833498Z 2022-01-11T22:28:40.766244Z 2022-01-11T22:28:10.580418Z 2022-01-11T22:28:43.921700Z 2022-01-11T22:28:09.878225Z 2022-01-11T22:28:28.858285Z 2022-01-11T22:27:55.908527Z 2022-01-11T22:28:27.633085Z 2022-01-11T22:28:30.102098Z 2022-01-11T22:27:58.099497Z 2022-01-11T22:27:37.795956Z 2022-01-11T22:27:57.607857Z 2022-01-11T22:27:37.186143Z 2022-01-11T22:27:55.749992Z 2022-01-11T22:27:36.053215Z 2022-01-11T22:27:50.313753Z 2022-01-11T22:27:35.078652Z 2022-01-11T22:27:49.334050Z 2022-01-11T22:28:21.260412Z 2022-01-11T22:27:48.658304Z 2022-01-11T22:29:33.006601Z 2022-01-11T22:27:44.225279Z 2022-01-11T22:27:42.294759Z 2022-01-11T22:27:45.107774Z 2022-01-11T22:28:53.394206Z 2022-01-11T22:27:34.335263Z 2022-01-11T22:29:24.004830Z 2022-01-11T22:27:38.145027Z 2022-01-11T22:29:12.682058Z 2022-01-11T22:27:40.542166Z 2022-01-11T22:29:03.234290Z 2022-01-11T22:27:27.681269Z 2022-01-11T22:27:33.624993Z 2022-01-11T22:27:26.403540Z 2022-01-11T22:29:04.676513Z 2022-01-11T22:28:51.918688Z 2022-01-11T22:29:05.615602Z 2022-01-11T22:28:52.674887Z 2022-01-11T22:29:05.293517Z 2022-01-11T22:28:51.711960Z 2022-01-11T22:29:04.420652Z 2022-01-11T22:28:49.530074Z 2022-01-11T22:29:05.181021Z 2022-01-11T22:29:03.147377Z 2022-01-11T22:29:01.434623Z 2022-01-11T22:28:50.338337Z 2022-01-11T22:29:01.957271Z 2022-01-11T22:28:51.657894Z 2022-01-11T22:29:04.405902Z 2022-01-11T22:28:56.181055Z 2022-01-11T22:28:49.929511Z 2022-01-11T22:28:59.887102Z 2022-01-11T22:28:51.410150Z 2022-01-11T22:28:58.856051Z 2022-01-11T22:28:46.221330Z 2022-01-11T22:29:02.457493Z 2022-01-11T22:28:51.024995Z 2022-01-11T22:29:02.142504Z 2022-01-11T22:28:46.490753Z 2022-01-11T22:29:02.263023Z 2022-01-11T22:28:43.321676Z 2022-01-11T22:28:58.003781Z 2022-01-11T22:28:48.707438Z 2022-01-11T22:29:02.382973Z 2022-01-11T22:28:46.036845Z 2022-01-11T22:28:57.195379Z 2022-01-11T22:28:46.629398Z 2022-01-11T22:28:55.252238Z 2022-01-11T22:28:48.169363Z 2022-01-11T22:28:54.965605Z 2022-01-11T22:28:43.544410Z 2022-01-11T22:28:51.530088Z 2022-01-11T22:28:40.373454Z 2022-01-11T22:28:52.947578Z 2022-01-11T22:28:40.902301Z 2022-01-11T22:28:48.409194Z 2022-01-11T22:28:42.658332Z 2022-01-11T22:28:54.255053Z 2022-01-11T22:28:41.807352Z 2022-01-11T22:28:52.038959Z 2022-01-11T22:28:41.317791Z 2022-01-11T22:28:55.578424Z 2022-01-11T22:28:35.753174Z 2022-01-11T22:28:47.059126Z 2022-01-11T22:28:37.610156Z 2022-01-11T22:28:46.973081Z 2022-01-11T22:28:40.122348Z 2022-01-11T22:28:43.101219Z 2022-01-11T22:28:34.865404Z 2022-01-11T22:28:48.360279Z 2022-01-11T22:28:34.039929Z 2022-01-11T22:28:42.245266Z 2022-01-11T22:28:30.048408Z 2022-01-11T22:28:43.192241Z 2022-01-11T22:28:30.595419Z 2022-01-11T22:28:44.176830Z 2022-01-11T22:28:27.642506Z 2022-01-11T22:28:44.269273Z 2022-01-11T22:28:23.839580Z 2022-01-11T22:33:01.106631Z 2022-01-11T22:33:02.515657Z 2022-01-11T22:33:00.840121Z 2022-01-11T22:33:00.945526Z 2022-01-11T22:33:02.651927Z 2022-01-11T22:33:01.935603Z 2022-01-11T22:33:02.577910Z 2022-01-11T22:33:01.446781Z 2022-01-11T22:33:01.292214Z 2022-01-11T22:33:01.194571Z 2022-01-11T22:33:01.486931Z 2022-01-11T22:32:59.547592Z 2022-01-11T22:33:01.650170Z 2022-01-11T22:33:01.330702Z 2022-01-11T22:33:00.284044Z 2022-01-11T22:33:00.838616Z 2022-01-11T22:32:55.460260Z 2022-01-11T22:32:58.403189Z 2022-01-11T22:33:00.217231Z 2022-01-11T22:32:58.089996Z 2022-01-11T22:32:55.574744Z 2022-01-11T22:33:00.733766Z 2022-01-11T22:33:00.849444Z 2022-01-11T22:32:57.678812Z 2022-01-11T22:32:59.506549Z 2022-01-11T22:32:58.568563Z 2022-01-11T22:33:00.663593Z 2022-01-11T22:32:58.554473Z 2022-01-11T22:32:57.086122Z 2022-01-11T22:32:58.555978Z 2022-01-11T22:32:56.056328Z 2022-01-11T22:33:00.601079Z 2022-01-11T22:32:53.910316Z 2022-01-11T22:32:59.324967Z 2022-01-11T22:32:58.766407Z 2022-01-11T22:32:56.527863Z 2022-01-11T22:32:58.404501Z 2022-01-11T22:32:51.111986Z 2022-01-11T22:32:52.069385Z 2022-01-11T22:32:57.992458Z 2022-01-11T22:32:49.180280Z 2022-01-11T22:32:51.601184Z 2022-01-11T22:32:55.314134Z 2022-01-11T22:32:57.246816Z 2022-01-11T22:32:51.272558Z 2022-01-11T22:32:59.756758Z 2022-01-11T22:32:49.588300Z 2022-01-11T22:32:52.746492Z 2022-01-11T22:32:45.498428Z 2022-01-11T22:32:49.646401Z 2022-01-11T22:32:45.924396Z 2022-01-11T22:32:50.059978Z 2022-01-11T22:32:40.592197Z 2022-01-11T22:32:43.419002Z 2022-01-11T22:32:51.457548Z 2022-01-11T22:32:41.891559Z 2022-01-11T22:32:31.118273Z 2022-01-11T22:32:37.703396Z 2022-01-11T22:32:42.705426Z 2022-01-11T22:32:48.018216Z 2022-01-11T22:32:35.265674Z 2022-01-11T22:32:48.975457Z 2022-01-11T22:32:39.464349Z 2022-01-11T22:32:42.687667Z 2022-01-11T22:32:36.123382Z 2022-01-11T22:32:37.808531Z 2022-01-11T22:32:31.928088Z 2022-01-11T22:32:38.105448Z 2022-01-11T22:32:32.400420Z 2022-01-11T22:32:41.928886Z 2022-01-11T22:32:30.265005Z 2022-01-11T22:32:40.631421Z 2022-01-11T22:32:26.488821Z 2022-01-11T22:32:36.819304Z 2022-01-11T22:32:26.729345Z 2022-01-11T22:32:33.999427Z 2022-01-11T22:32:01.134256Z 2022-01-11T22:31:53.280403Z 2022-01-11T22:32:01.012943Z 2022-01-11T22:31:52.062336Z 2022-01-11T22:31:58.000410Z 2022-01-11T22:31:52.455684Z 2022-01-11T22:32:04.379137Z 2022-01-11T22:31:53.086180Z 2022-01-11T22:31:53.586843Z 2022-01-11T22:31:50.645797Z 2022-01-11T22:31:53.274641Z 2022-01-11T22:31:49.124322Z 2022-01-11T22:31:47.237979Z 2022-01-11T22:31:50.474818Z 2022-01-11T22:31:49.914905Z 2022-01-11T22:31:48.422110Z 2022-01-11T22:31:48.551258Z 2022-01-11T22:31:41.905034Z 2022-01-11T22:31:38.682498Z 2022-01-11T22:31:42.332087Z 2022-01-11T22:31:35.990565Z 2022-01-11T22:31:39.934741Z 2022-01-11T22:31:30.948812Z 2022-01-11T22:31:37.788797Z 2022-01-11T22:36:57.444025Z 2022-01-11T22:36:58.591452Z 2022-01-11T22:36:57.776553Z 2022-01-11T22:36:58.617048Z 2022-01-11T22:36:57.671510Z 2022-01-11T22:36:58.304807Z 2022-01-11T22:36:57.673053Z 2022-01-11T22:36:56.564551Z 2022-01-11T22:36:56.276742Z 2022-01-11T22:36:58.396290Z 2022-01-11T22:36:58.162099Z 2022-01-11T22:36:57.155197Z 2022-01-11T22:36:58.070180Z 2022-01-11T22:36:58.636499Z 2022-01-11T22:36:55.264184Z 2022-01-11T22:36:57.511699Z 2022-01-11T22:36:53.925862Z 2022-01-11T22:36:55.190747Z 2022-01-11T22:36:54.308954Z 2022-01-11T22:36:52.803443Z 2022-01-11T22:36:58.680052Z 2022-01-11T22:36:56.677596Z 2022-01-11T22:36:53.524481Z 2022-01-11T22:36:54.771772Z 2022-01-11T22:36:52.158283Z 2022-01-11T22:36:53.666854Z 2022-01-11T22:36:53.239392Z 2022-01-11T22:36:58.325282Z 2022-01-11T22:36:55.733858Z 2022-01-11T22:36:56.693359Z 2022-01-11T22:36:45.677700Z 2022-01-11T22:36:53.569949Z 2022-01-11T22:36:48.260534Z 2022-01-11T22:36:55.779701Z 2022-01-11T22:36:53.066167Z 2022-01-11T22:36:55.250020Z 2022-01-11T22:36:52.658118Z 2022-01-11T22:36:52.135783Z 2022-01-11T22:36:51.823550Z 2022-01-11T22:36:52.012295Z 2022-01-11T22:36:51.432208Z 2022-01-11T22:36:53.214462Z 2022-01-11T22:36:43.455059Z 2022-01-11T22:36:32.932470Z 2022-01-11T22:36:36.032036Z 2022-01-11T22:36:34.623386Z 2022-01-11T22:36:45.589177Z 2022-01-11T22:36:39.683283Z 2022-01-11T22:36:51.888299Z 2022-01-11T22:36:41.413032Z 2022-01-11T22:36:36.598607Z 2022-01-11T22:36:34.229109Z 2022-01-11T22:36:27.461220Z 2022-01-11T22:36:37.513835Z 2022-01-11T22:36:33.143455Z 2022-01-11T22:36:31.620278Z 2022-01-11T22:36:27.553577Z 2022-01-11T22:36:35.209617Z 2022-01-11T22:36:37.057572Z 2022-01-11T22:36:35.331016Z 2022-01-11T22:36:31.065521Z 2022-01-11T22:36:32.820234Z 2022-01-11T22:36:18.986613Z 2022-01-11T22:36:36.025321Z 2022-01-11T22:36:29.025432Z 2022-01-11T22:36:32.144428Z 2022-01-11T22:36:21.798350Z 2022-01-11T22:36:22.996207Z 2022-01-11T22:36:08.663804Z 2022-01-11T22:36:23.846160Z 2022-01-11T22:36:15.356822Z 2022-01-11T22:36:19.480201Z 2022-01-11T22:36:14.211800Z 2022-01-11T22:36:17.711977Z 2022-01-11T22:36:13.569572Z 2022-01-11T22:36:21.371033Z 2022-01-11T22:36:10.081227Z 2022-01-11T22:36:24.198371Z 2022-01-11T22:36:13.503962Z 2022-01-11T22:36:08.963422Z 2022-01-11T22:36:06.451493Z 2022-01-11T22:36:09.340128Z 2022-01-11T22:36:00.305505Z 2022-01-11T22:36:17.807556Z 2022-01-11T22:35:57.417534Z 2022-01-11T22:36:02.959252Z 2022-01-11T22:35:55.104352Z 2022-01-11T22:36:04.077030Z 2022-01-11T22:35:52.538368Z 2022-01-11T22:36:01.344484Z 2022-01-11T22:35:52.048480Z 2022-01-11T22:36:01.006685Z 2022-01-11T22:35:33.996885Z 2022-01-11T22:35:38.433101Z 2022-01-11T22:35:33.372648Z 2022-01-11T22:35:37.847130Z 2022-01-11T22:35:32.765571Z 2022-01-11T22:35:34.394120Z 2022-01-11T22:35:30.772392Z 2022-01-11T22:35:29.630335Z 2022-01-11T22:40:54.702896Z 2022-01-11T22:41:00.661975Z 2022-01-11T22:40:55.742966Z 2022-01-11T22:40:57.037179Z 2022-01-11T22:40:52.767945Z 2022-01-11T22:41:00.407132Z 2022-01-11T22:40:55.342288Z 2022-01-11T22:40:59.169128Z 2022-01-11T22:40:52.603269Z 2022-01-11T22:40:56.340420Z 2022-01-11T22:40:54.300416Z 2022-01-11T22:40:58.096806Z 2022-01-11T22:40:54.747098Z 2022-01-11T22:40:58.672230Z 2022-01-11T22:40:55.453727Z 2022-01-11T22:40:59.634979Z 2022-01-11T22:40:55.333515Z 2022-01-11T22:40:58.265237Z 2022-01-11T22:40:55.093261Z 2022-01-11T22:40:58.554685Z 2022-01-11T22:40:55.338058Z 2022-01-11T22:40:58.540379Z 2022-01-11T22:40:53.804703Z 2022-01-11T22:40:58.475454Z 2022-01-11T22:40:53.911486Z 2022-01-11T22:40:56.578094Z 2022-01-11T22:40:48.944062Z 2022-01-11T22:40:52.236605Z 2022-01-11T22:40:49.946077Z 2022-01-11T22:40:54.826323Z 2022-01-11T22:40:48.787151Z 2022-01-11T22:40:56.832113Z 2022-01-11T22:40:51.253859Z 2022-01-11T22:40:57.515026Z 2022-01-11T22:40:53.143575Z 2022-01-11T22:40:55.149817Z 2022-01-11T22:40:49.750197Z 2022-01-11T22:40:54.443091Z 2022-01-11T22:40:47.507549Z 2022-01-11T22:40:58.851587Z 2022-01-11T22:40:44.682718Z 2022-01-11T22:40:53.811181Z 2022-01-11T22:40:40.381444Z 2022-01-11T22:40:53.791385Z 2022-01-11T22:40:49.145887Z 2022-01-11T22:40:48.701208Z 2022-01-11T22:40:43.792745Z 2022-01-11T22:40:51.079179Z 2022-01-11T22:40:42.937484Z 2022-01-11T22:40:45.981270Z 2022-01-11T22:40:44.384452Z 2022-01-11T22:40:46.905122Z 2022-01-11T22:40:46.428469Z 2022-01-11T22:40:45.354047Z 2022-01-11T22:40:46.548797Z 2022-01-11T22:40:44.812180Z 2022-01-11T22:40:35.241878Z 2022-01-11T22:40:40.928205Z 2022-01-11T22:40:36.980286Z 2022-01-11T22:40:43.606045Z 2022-01-11T22:40:35.565350Z 2022-01-11T22:40:39.349207Z 2022-01-11T22:40:30.188523Z 2022-01-11T22:40:43.932678Z 2022-01-11T22:40:36.274434Z 2022-01-11T22:40:40.182185Z 2022-01-11T22:40:22.429679Z 2022-01-11T22:40:37.988284Z 2022-01-11T22:40:28.612411Z 2022-01-11T22:40:35.575338Z 2022-01-11T22:40:22.254966Z 2022-01-11T22:40:26.963278Z 2022-01-11T22:40:21.919459Z 2022-01-11T22:40:33.746367Z 2022-01-11T22:40:15.241587Z 2022-01-11T22:39:52.873078Z 2022-01-11T22:40:31.106195Z 2022-01-11T22:40:02.191280Z 2022-01-11T22:39:48.358386Z 2022-01-11T22:39:52.287727Z 2022-01-11T22:39:49.663386Z 2022-01-11T22:39:51.933360Z 2022-01-11T22:39:47.037393Z 2022-01-11T22:39:51.583428Z 2022-01-11T22:39:47.301853Z 2022-01-11T22:39:51.034661Z 2022-01-11T22:39:46.798240Z 2022-01-11T22:39:49.140086Z 2022-01-11T22:39:44.796259Z 2022-01-11T22:39:49.399667Z 2022-01-11T22:39:45.370391Z 2022-01-11T22:39:45.612506Z 2022-01-11T22:39:40.526856Z 2022-01-11T22:39:45.207668Z 2022-01-11T22:39:40.009503Z 2022-01-11T22:39:44.358179Z 2022-01-11T22:39:37.125500Z 2022-01-11T22:39:32.462293Z 2022-01-11T22:39:27.097751Z 2022-01-11T22:39:31.378239Z 2022-01-11T22:45:00.525843Z 2022-01-11T22:45:01.692238Z 2022-01-11T22:45:01.984383Z 2022-01-11T22:45:01.215902Z 2022-01-11T22:45:01.623689Z 2022-01-11T22:45:00.334893Z 2022-01-11T22:45:01.993062Z 2022-01-11T22:44:58.740784Z 2022-01-11T22:45:01.223949Z 2022-01-11T22:45:00.036504Z 2022-01-11T22:45:01.491309Z 2022-01-11T22:45:01.287674Z 2022-01-11T22:45:00.575056Z 2022-01-11T22:45:01.771889Z 2022-01-11T22:45:00.891152Z 2022-01-11T22:45:01.333537Z 2022-01-11T22:45:01.424144Z 2022-01-11T22:45:01.476699Z 2022-01-11T22:44:58.579219Z 2022-01-11T22:45:00.344013Z 2022-01-11T22:44:57.597425Z 2022-01-11T22:44:58.899282Z 2022-01-11T22:45:00.127833Z 2022-01-11T22:44:57.636058Z 2022-01-11T22:44:53.568273Z 2022-01-11T22:44:56.247835Z 2022-01-11T22:44:55.746662Z 2022-01-11T22:44:53.018847Z 2022-01-11T22:44:52.476724Z 2022-01-11T22:44:56.102698Z 2022-01-11T22:44:51.400012Z 2022-01-11T22:44:59.566670Z 2022-01-11T22:44:49.939663Z 2022-01-11T22:44:52.057017Z 2022-01-11T22:44:52.358551Z 2022-01-11T22:44:55.871653Z 2022-01-11T22:44:49.250478Z 2022-01-11T22:44:48.517615Z 2022-01-11T22:44:49.947308Z 2022-01-11T22:44:52.340313Z 2022-01-11T22:44:54.638489Z 2022-01-11T22:44:51.213468Z 2022-01-11T22:44:50.882606Z 2022-01-11T22:44:47.453223Z 2022-01-11T22:44:40.839588Z 2022-01-11T22:44:53.595369Z 2022-01-11T22:44:50.763913Z 2022-01-11T22:44:42.161379Z 2022-01-11T22:44:45.006451Z 2022-01-11T22:44:47.022289Z 2022-01-11T22:44:49.083438Z 2022-01-11T22:44:44.619676Z 2022-01-11T22:44:46.791644Z 2022-01-11T22:44:45.600605Z 2022-01-11T22:44:47.417686Z 2022-01-11T22:44:47.994224Z 2022-01-11T22:44:44.842555Z 2022-01-11T22:44:51.013160Z 2022-01-11T22:44:42.325522Z 2022-01-11T22:44:40.164406Z 2022-01-11T22:44:46.039407Z 2022-01-11T22:44:41.108279Z 2022-01-11T22:44:34.142528Z 2022-01-11T22:44:38.619068Z 2022-01-11T22:44:40.703175Z 2022-01-11T22:44:39.180999Z 2022-01-11T22:44:39.665637Z 2022-01-11T22:44:35.833327Z 2022-01-11T22:44:37.624204Z 2022-01-11T22:44:41.070920Z 2022-01-11T22:44:27.879384Z 2022-01-11T22:44:35.056539Z 2022-01-11T22:44:33.833230Z 2022-01-11T22:44:20.735277Z 2022-01-11T22:44:37.435956Z 2022-01-11T22:44:30.343178Z 2022-01-11T22:44:28.144120Z 2022-01-11T22:44:23.722625Z 2022-01-11T22:44:30.281427Z 2022-01-11T22:44:26.273193Z 2022-01-11T22:44:26.714377Z 2022-01-11T22:44:09.711115Z 2022-01-11T22:44:13.853713Z 2022-01-11T22:44:05.115283Z 2022-01-11T22:44:13.793645Z 2022-01-11T22:44:10.706422Z 2022-01-11T22:44:12.429485Z 2022-01-11T22:44:09.655356Z 2022-01-11T22:43:43.332481Z 2022-01-11T22:43:39.435307Z 2022-01-11T22:43:42.471617Z 2022-01-11T22:43:39.315638Z 2022-01-11T22:43:40.304582Z 2022-01-11T22:43:38.426928Z 2022-01-11T22:43:37.354546Z 2022-01-11T22:43:35.243277Z 2022-01-11T22:43:31.763307Z 2022-01-11T22:43:34.534250Z 2022-01-11T22:43:28.700346Z 2022-01-11T22:43:28.647432Z", + "finished_diff": 98.8531494, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 130.711636, + "mean": 47.76437906399998, + "min": 10.660373 + }, + "jobs_events_duration": { + "max": 6.624046, + "mean": 2.841105231999999, + "min": 0.331843 + }, + "jobs_events_lag": { + "max": -1.315495, + "mean": -4.891911439999995, + "min": -8.356099 + }, + "jobs_waiting": { + "max": 47.706787, + "mean": 25.578230398000006, + "min": 0.579451 + }, + "modified": "2022-01-11T22:27:35.093089Z 2022-01-11T22:27:35.047072Z 2022-01-11T22:27:35.010727Z 2022-01-11T22:27:34.978689Z 2022-01-11T22:27:34.887427Z 2022-01-11T22:27:34.822428Z 2022-01-11T22:27:34.687358Z 2022-01-11T22:27:34.514360Z 2022-01-11T22:27:34.407174Z 2022-01-11T22:27:34.310639Z 2022-01-11T22:27:19.541094Z 2022-01-11T22:27:19.301318Z 2022-01-11T22:27:18.937855Z 2022-01-11T22:27:18.625186Z 2022-01-11T22:27:18.300319Z 2022-01-11T22:27:18.132735Z 2022-01-11T22:27:17.992480Z 2022-01-11T22:27:17.804434Z 2022-01-11T22:27:17.614622Z 2022-01-11T22:27:17.555364Z 2022-01-11T22:27:17.482363Z 2022-01-11T22:27:17.373387Z 2022-01-11T22:27:17.201893Z 2022-01-11T22:27:16.774403Z 2022-01-11T22:27:16.491737Z 2022-01-11T22:27:14.153238Z 2022-01-11T22:27:14.066367Z 2022-01-11T22:27:13.956352Z 2022-01-11T22:27:13.860383Z 2022-01-11T22:27:13.719372Z 2022-01-11T22:27:13.584162Z 2022-01-11T22:27:12.585097Z 2022-01-11T22:27:12.511663Z 2022-01-11T22:27:12.021206Z 2022-01-11T22:27:11.990378Z 2022-01-11T22:28:11.930550Z 2022-01-11T22:28:11.872068Z 2022-01-11T22:28:11.810475Z 2022-01-11T22:28:11.745390Z 2022-01-11T22:28:11.682506Z 2022-01-11T22:28:11.624444Z 2022-01-11T22:28:11.573410Z 2022-01-11T22:27:39.053558Z 2022-01-11T22:28:11.530718Z 2022-01-11T22:28:11.471439Z 2022-01-11T22:27:38.818382Z 2022-01-11T22:27:38.772345Z 2022-01-11T22:27:38.722334Z 2022-01-11T22:27:38.677775Z 2022-01-11T22:28:11.411638Z 2022-01-11T22:27:38.629162Z 2022-01-11T22:27:38.591096Z 2022-01-11T22:27:38.510018Z 2022-01-11T22:27:38.427895Z 2022-01-11T22:27:38.367232Z 2022-01-11T22:27:38.331594Z 2022-01-11T22:27:38.291210Z 2022-01-11T22:27:38.232270Z 2022-01-11T22:27:38.181685Z 2022-01-11T22:27:38.116216Z 2022-01-11T22:27:38.080464Z 2022-01-11T22:27:38.040827Z 2022-01-11T22:27:38.006237Z 2022-01-11T22:27:37.970472Z 2022-01-11T22:27:37.937248Z 2022-01-11T22:27:37.886176Z 2022-01-11T22:27:37.840173Z 2022-01-11T22:27:37.803280Z 2022-01-11T22:27:37.742553Z 2022-01-11T22:27:37.682696Z 2022-01-11T22:27:37.624263Z 2022-01-11T22:27:37.560293Z 2022-01-11T22:27:37.498949Z 2022-01-11T22:27:37.437675Z 2022-01-11T22:27:37.376941Z 2022-01-11T22:27:37.319235Z 2022-01-11T22:27:37.256604Z 2022-01-11T22:27:37.189226Z 2022-01-11T22:27:37.063280Z 2022-01-11T22:27:36.971313Z 2022-01-11T22:27:36.896333Z 2022-01-11T22:27:36.802207Z 2022-01-11T22:27:36.704385Z 2022-01-11T22:27:36.566371Z 2022-01-11T22:27:36.444903Z 2022-01-11T22:27:36.387962Z 2022-01-11T22:27:36.283580Z 2022-01-11T22:27:36.195379Z 2022-01-11T22:27:36.109235Z 2022-01-11T22:27:35.975469Z 2022-01-11T22:27:35.815608Z 2022-01-11T22:27:35.694495Z 2022-01-11T22:27:35.578590Z 2022-01-11T22:27:35.470542Z 2022-01-11T22:27:35.388472Z 2022-01-11T22:27:35.310062Z 2022-01-11T22:27:35.262733Z 2022-01-11T22:27:35.205190Z 2022-01-11T22:27:35.165935Z 2022-01-11T22:27:35.132201Z 2022-01-11T22:32:08.224463Z 2022-01-11T22:32:08.162581Z 2022-01-11T22:32:08.096368Z 2022-01-11T22:32:08.027518Z 2022-01-11T22:32:07.951412Z 2022-01-11T22:32:07.865443Z 2022-01-11T22:32:07.786700Z 2022-01-11T22:32:07.707676Z 2022-01-11T22:32:07.615451Z 2022-01-11T22:32:07.530559Z 2022-01-11T22:32:07.428656Z 2022-01-11T22:32:07.327398Z 2022-01-11T22:32:07.211790Z 2022-01-11T22:32:07.108502Z 2022-01-11T22:32:06.979477Z 2022-01-11T22:32:06.864772Z 2022-01-11T22:31:43.709622Z 2022-01-11T22:31:43.597737Z 2022-01-11T22:31:43.504512Z 2022-01-11T22:31:43.405822Z 2022-01-11T22:31:43.308243Z 2022-01-11T22:31:43.207407Z 2022-01-11T22:31:43.148485Z 2022-01-11T22:31:43.105485Z 2022-01-11T22:31:43.061450Z 2022-01-11T22:31:43.009448Z 2022-01-11T22:31:42.968450Z 2022-01-11T22:31:42.910895Z 2022-01-11T22:31:42.874903Z 2022-01-11T22:31:42.818668Z 2022-01-11T22:31:42.777992Z 2022-01-11T22:31:42.738445Z 2022-01-11T22:31:42.676232Z 2022-01-11T22:31:42.594629Z 2022-01-11T22:31:42.500440Z 2022-01-11T22:31:42.364800Z 2022-01-11T22:31:42.153610Z 2022-01-11T22:31:42.272692Z 2022-01-11T22:31:42.038496Z 2022-01-11T22:31:41.905581Z 2022-01-11T22:31:41.788740Z 2022-01-11T22:31:41.642929Z 2022-01-11T22:31:41.508810Z 2022-01-11T22:31:41.425549Z 2022-01-11T22:31:41.370668Z 2022-01-11T22:31:41.309664Z 2022-01-11T22:31:41.220463Z 2022-01-11T22:31:41.137385Z 2022-01-11T22:31:41.063399Z 2022-01-11T22:31:40.980990Z 2022-01-11T22:31:40.931517Z 2022-01-11T22:31:40.880424Z 2022-01-11T22:31:40.842056Z 2022-01-11T22:31:40.804916Z 2022-01-11T22:31:40.765092Z 2022-01-11T22:31:40.724282Z 2022-01-11T22:31:40.680331Z 2022-01-11T22:31:40.624567Z 2022-01-11T22:31:40.555481Z 2022-01-11T22:31:40.467127Z 2022-01-11T22:31:40.333561Z 2022-01-11T22:31:40.202658Z 2022-01-11T22:31:40.117667Z 2022-01-11T22:31:40.019179Z 2022-01-11T22:31:39.901271Z 2022-01-11T22:31:39.737128Z 2022-01-11T22:31:39.564004Z 2022-01-11T22:31:39.335530Z 2022-01-11T22:31:39.063996Z 2022-01-11T22:31:38.887561Z 2022-01-11T22:31:38.665610Z 2022-01-11T22:31:38.532537Z 2022-01-11T22:31:38.441183Z 2022-01-11T22:31:38.323184Z 2022-01-11T22:31:38.214889Z 2022-01-11T22:31:38.109813Z 2022-01-11T22:31:24.563623Z 2022-01-11T22:31:24.410345Z 2022-01-11T22:31:24.311418Z 2022-01-11T22:31:24.223774Z 2022-01-11T22:31:24.110967Z 2022-01-11T22:31:24.012488Z 2022-01-11T22:31:23.900408Z 2022-01-11T22:31:23.844266Z 2022-01-11T22:31:23.748671Z 2022-01-11T22:31:23.585859Z 2022-01-11T22:31:23.507442Z 2022-01-11T22:31:23.336069Z 2022-01-11T22:31:23.235405Z 2022-01-11T22:31:23.117732Z 2022-01-11T22:31:22.979170Z 2022-01-11T22:31:22.905627Z 2022-01-11T22:31:22.752095Z 2022-01-11T22:31:20.754584Z 2022-01-11T22:31:20.687189Z 2022-01-11T22:31:20.525311Z 2022-01-11T22:31:20.351070Z 2022-01-11T22:31:19.594376Z 2022-01-11T22:31:19.560100Z 2022-01-11T22:31:19.523647Z 2022-01-11T22:35:57.047286Z 2022-01-11T22:35:56.991736Z 2022-01-11T22:35:56.930361Z 2022-01-11T22:35:56.874397Z 2022-01-11T22:35:56.816570Z 2022-01-11T22:35:56.763438Z 2022-01-11T22:35:56.700540Z 2022-01-11T22:35:56.631412Z 2022-01-11T22:35:56.544651Z 2022-01-11T22:35:56.451506Z 2022-01-11T22:35:56.333992Z 2022-01-11T22:35:56.232547Z 2022-01-11T22:35:56.137646Z 2022-01-11T22:35:56.052086Z 2022-01-11T22:35:55.949593Z 2022-01-11T22:35:55.839452Z 2022-01-11T22:35:55.738465Z 2022-01-11T22:35:55.634498Z 2022-01-11T22:35:55.527014Z 2022-01-11T22:35:55.427578Z 2022-01-11T22:35:55.341545Z 2022-01-11T22:35:55.288659Z 2022-01-11T22:35:55.231419Z 2022-01-11T22:35:55.181096Z 2022-01-11T22:35:55.142102Z 2022-01-11T22:35:55.058621Z 2022-01-11T22:35:55.006087Z 2022-01-11T22:35:54.938715Z 2022-01-11T22:35:54.880587Z 2022-01-11T22:35:54.821438Z 2022-01-11T22:35:54.765579Z 2022-01-11T22:35:54.709492Z 2022-01-11T22:35:54.640697Z 2022-01-11T22:35:54.580549Z 2022-01-11T22:35:54.511397Z 2022-01-11T22:35:54.446598Z 2022-01-11T22:35:54.411763Z 2022-01-11T22:35:54.336672Z 2022-01-11T22:35:54.275699Z 2022-01-11T22:35:54.210674Z 2022-01-11T22:35:54.152793Z 2022-01-11T22:35:54.082610Z 2022-01-11T22:35:29.211427Z 2022-01-11T22:35:29.145060Z 2022-01-11T22:35:29.107340Z 2022-01-11T22:35:29.067422Z 2022-01-11T22:35:29.028574Z 2022-01-11T22:35:28.994634Z 2022-01-11T22:35:54.041985Z 2022-01-11T22:35:28.954648Z 2022-01-11T22:35:28.776501Z 2022-01-11T22:35:28.718844Z 2022-01-11T22:35:28.626642Z 2022-01-11T22:35:28.591629Z 2022-01-11T22:35:28.552101Z 2022-01-11T22:35:28.517652Z 2022-01-11T22:35:28.482870Z 2022-01-11T22:35:28.448439Z 2022-01-11T22:35:28.415208Z 2022-01-11T22:35:28.381897Z 2022-01-11T22:35:28.349386Z 2022-01-11T22:35:28.317273Z 2022-01-11T22:35:28.285407Z 2022-01-11T22:35:28.252078Z 2022-01-11T22:35:28.216239Z 2022-01-11T22:35:28.182447Z 2022-01-11T22:35:28.148840Z 2022-01-11T22:35:28.114945Z 2022-01-11T22:35:28.074046Z 2022-01-11T22:35:28.034086Z 2022-01-11T22:35:27.994605Z 2022-01-11T22:35:27.953334Z 2022-01-11T22:35:27.912418Z 2022-01-11T22:35:27.875248Z 2022-01-11T22:35:27.826118Z 2022-01-11T22:35:27.790603Z 2022-01-11T22:35:27.742442Z 2022-01-11T22:35:27.699683Z 2022-01-11T22:35:27.658379Z 2022-01-11T22:35:27.569203Z 2022-01-11T22:35:27.506247Z 2022-01-11T22:35:27.466335Z 2022-01-11T22:35:27.413346Z 2022-01-11T22:35:27.339830Z 2022-01-11T22:35:27.205684Z 2022-01-11T22:35:27.068676Z 2022-01-11T22:35:26.885762Z 2022-01-11T22:35:26.610897Z 2022-01-11T22:35:26.508305Z 2022-01-11T22:35:26.439348Z 2022-01-11T22:35:26.067589Z 2022-01-11T22:35:25.984738Z 2022-01-11T22:35:17.621364Z 2022-01-11T22:35:17.510015Z 2022-01-11T22:35:17.383203Z 2022-01-11T22:35:17.227683Z 2022-01-11T22:35:17.048994Z 2022-01-11T22:35:16.917397Z 2022-01-11T22:35:16.070643Z 2022-01-11T22:35:16.035250Z 2022-01-11T22:39:38.461389Z 2022-01-11T22:39:38.382276Z 2022-01-11T22:39:38.323282Z 2022-01-11T22:39:38.270317Z 2022-01-11T22:39:38.222110Z 2022-01-11T22:39:38.180907Z 2022-01-11T22:39:38.126697Z 2022-01-11T22:39:38.086689Z 2022-01-11T22:39:38.050563Z 2022-01-11T22:39:38.003629Z 2022-01-11T22:39:37.958889Z 2022-01-11T22:39:37.902172Z 2022-01-11T22:39:37.859252Z 2022-01-11T22:39:37.802356Z 2022-01-11T22:39:37.728305Z 2022-01-11T22:39:37.645324Z 2022-01-11T22:39:37.569237Z 2022-01-11T22:39:37.492096Z 2022-01-11T22:39:37.416326Z 2022-01-11T22:39:37.332310Z 2022-01-11T22:39:37.253396Z 2022-01-11T22:39:37.198620Z 2022-01-11T22:39:37.140413Z 2022-01-11T22:39:37.095386Z 2022-01-11T22:39:37.047407Z 2022-01-11T22:39:36.996403Z 2022-01-11T22:39:36.899581Z 2022-01-11T22:39:36.952261Z 2022-01-11T22:39:36.779394Z 2022-01-11T22:39:36.690330Z 2022-01-11T22:39:36.600307Z 2022-01-11T22:39:36.516333Z 2022-01-11T22:39:36.427439Z 2022-01-11T22:39:36.327277Z 2022-01-11T22:39:36.238346Z 2022-01-11T22:39:36.149507Z 2022-01-11T22:39:36.065349Z 2022-01-11T22:39:35.975431Z 2022-01-11T22:39:35.881296Z 2022-01-11T22:39:35.824286Z 2022-01-11T22:39:35.769443Z 2022-01-11T22:39:35.714264Z 2022-01-11T22:39:35.651332Z 2022-01-11T22:39:35.577179Z 2022-01-11T22:39:35.494272Z 2022-01-11T22:39:35.416221Z 2022-01-11T22:39:35.327345Z 2022-01-11T22:39:35.236468Z 2022-01-11T22:39:35.153231Z 2022-01-11T22:39:35.062514Z 2022-01-11T22:39:34.981296Z 2022-01-11T22:39:34.893362Z 2022-01-11T22:39:34.807307Z 2022-01-11T22:39:34.735259Z 2022-01-11T22:39:34.660340Z 2022-01-11T22:39:34.575504Z 2022-01-11T22:39:34.492423Z 2022-01-11T22:39:34.424988Z 2022-01-11T22:39:34.380173Z 2022-01-11T22:39:34.345068Z 2022-01-11T22:39:34.294328Z 2022-01-11T22:39:34.221496Z 2022-01-11T22:39:34.095594Z 2022-01-11T22:39:33.968476Z 2022-01-11T22:39:33.926341Z 2022-01-11T22:39:33.877675Z 2022-01-11T22:39:33.831240Z 2022-01-11T22:39:33.783441Z 2022-01-11T22:39:33.728232Z 2022-01-11T22:39:33.667847Z 2022-01-11T22:39:33.634554Z 2022-01-11T22:39:33.594214Z 2022-01-11T22:39:33.547128Z 2022-01-11T22:39:33.509921Z 2022-01-11T22:39:33.465325Z 2022-01-11T22:39:23.559535Z 2022-01-11T22:39:33.423081Z 2022-01-11T22:39:23.327826Z 2022-01-11T22:39:23.190617Z 2022-01-11T22:39:22.958518Z 2022-01-11T22:39:22.766389Z 2022-01-11T22:39:22.511483Z 2022-01-11T22:39:22.272323Z 2022-01-11T22:39:22.119470Z 2022-01-11T22:39:21.830490Z 2022-01-11T22:39:21.622984Z 2022-01-11T22:39:21.415239Z 2022-01-11T22:39:21.246861Z 2022-01-11T22:39:21.167636Z 2022-01-11T22:39:20.937994Z 2022-01-11T22:39:20.693214Z 2022-01-11T22:39:20.423322Z 2022-01-11T22:39:20.203238Z 2022-01-11T22:39:19.872462Z 2022-01-11T22:39:19.531367Z 2022-01-11T22:39:19.205639Z 2022-01-11T22:39:18.909313Z 2022-01-11T22:39:15.739735Z 2022-01-11T22:39:15.637291Z 2022-01-11T22:39:15.469936Z 2022-01-11T22:44:04.500172Z 2022-01-11T22:44:04.426840Z 2022-01-11T22:44:04.362078Z 2022-01-11T22:44:04.296509Z 2022-01-11T22:44:04.208637Z 2022-01-11T22:44:04.123667Z 2022-01-11T22:44:04.052738Z 2022-01-11T22:44:03.987980Z 2022-01-11T22:44:03.936451Z 2022-01-11T22:44:03.881482Z 2022-01-11T22:44:03.820180Z 2022-01-11T22:44:03.767607Z 2022-01-11T22:44:03.709549Z 2022-01-11T22:44:03.653673Z 2022-01-11T22:44:03.591835Z 2022-01-11T22:44:03.505815Z 2022-01-11T22:44:03.397550Z 2022-01-11T22:44:03.289936Z 2022-01-11T22:44:03.190483Z 2022-01-11T22:44:03.075962Z 2022-01-11T22:43:34.284993Z 2022-01-11T22:44:02.969823Z 2022-01-11T22:44:02.874410Z 2022-01-11T22:43:34.246300Z 2022-01-11T22:43:34.212055Z 2022-01-11T22:43:34.178212Z 2022-01-11T22:43:34.142413Z 2022-01-11T22:43:34.106996Z 2022-01-11T22:43:34.069113Z 2022-01-11T22:43:34.035215Z 2022-01-11T22:43:33.993151Z 2022-01-11T22:44:02.772176Z 2022-01-11T22:43:33.927442Z 2022-01-11T22:43:33.875925Z 2022-01-11T22:43:33.839105Z 2022-01-11T22:43:33.806741Z 2022-01-11T22:43:33.774144Z 2022-01-11T22:43:33.734140Z 2022-01-11T22:43:33.684247Z 2022-01-11T22:43:33.645411Z 2022-01-11T22:43:33.597373Z 2022-01-11T22:43:33.552845Z 2022-01-11T22:43:33.477747Z 2022-01-11T22:43:33.417344Z 2022-01-11T22:43:33.382228Z 2022-01-11T22:43:33.343536Z 2022-01-11T22:43:33.300027Z 2022-01-11T22:43:33.263169Z 2022-01-11T22:43:33.207701Z 2022-01-11T22:43:33.153435Z 2022-01-11T22:43:33.049624Z 2022-01-11T22:43:32.943880Z 2022-01-11T22:43:32.871395Z 2022-01-11T22:43:32.820200Z 2022-01-11T22:43:32.782528Z 2022-01-11T22:43:32.741257Z 2022-01-11T22:43:32.707090Z 2022-01-11T22:43:32.663457Z 2022-01-11T22:43:32.619951Z 2022-01-11T22:43:32.572161Z 2022-01-11T22:43:32.536015Z 2022-01-11T22:43:32.502725Z 2022-01-11T22:43:32.449799Z 2022-01-11T22:43:32.323597Z 2022-01-11T22:43:32.387590Z 2022-01-11T22:43:32.285639Z 2022-01-11T22:43:32.229954Z 2022-01-11T22:43:32.148015Z 2022-01-11T22:43:32.043373Z 2022-01-11T22:43:31.946079Z 2022-01-11T22:43:31.861556Z 2022-01-11T22:43:31.789575Z 2022-01-11T22:43:31.657174Z 2022-01-11T22:43:31.586904Z 2022-01-11T22:43:31.476104Z 2022-01-11T22:43:31.424537Z 2022-01-11T22:43:31.328053Z 2022-01-11T22:43:31.271081Z 2022-01-11T22:43:31.221119Z 2022-01-11T22:43:31.089407Z 2022-01-11T22:43:30.875634Z 2022-01-11T22:43:30.743494Z 2022-01-11T22:43:30.576234Z 2022-01-11T22:43:30.429872Z 2022-01-11T22:43:30.361731Z 2022-01-11T22:43:30.259670Z 2022-01-11T22:43:30.028674Z 2022-01-11T22:43:29.905813Z 2022-01-11T22:43:20.617937Z 2022-01-11T22:43:20.413036Z 2022-01-11T22:43:20.269667Z 2022-01-11T22:43:19.974263Z 2022-01-11T22:43:19.622430Z 2022-01-11T22:43:19.412623Z 2022-01-11T22:43:19.215530Z 2022-01-11T22:43:17.405032Z 2022-01-11T22:43:17.370641Z 2022-01-11T22:43:17.335935Z 2022-01-11T22:43:16.178381Z 2022-01-11T22:43:16.145004Z", + "modified_diff": 44.199929, + "started": "2022-01-11T22:27:42.019112Z 2022-01-11T22:27:41.672272Z 2022-01-11T22:27:41.492527Z 2022-01-11T22:27:41.117612Z 2022-01-11T22:27:41.115361Z 2022-01-11T22:27:40.582129Z 2022-01-11T22:27:40.523558Z 2022-01-11T22:27:39.879128Z 2022-01-11T22:27:39.756037Z 2022-01-11T22:27:39.499229Z 2022-01-11T22:27:27.740644Z 2022-01-11T22:27:26.369876Z 2022-01-11T22:27:26.389379Z 2022-01-11T22:27:25.887521Z 2022-01-11T22:27:25.919683Z 2022-01-11T22:27:25.044710Z 2022-01-11T22:27:24.654710Z 2022-01-11T22:27:24.097852Z 2022-01-11T22:27:24.220254Z 2022-01-11T22:27:23.123456Z 2022-01-11T22:27:22.974687Z 2022-01-11T22:27:22.294965Z 2022-01-11T22:27:22.508478Z 2022-01-11T22:27:21.482668Z 2022-01-11T22:27:21.227636Z 2022-01-11T22:27:15.561469Z 2022-01-11T22:27:15.273672Z 2022-01-11T22:27:15.033986Z 2022-01-11T22:27:15.134652Z 2022-01-11T22:27:14.653381Z 2022-01-11T22:27:14.620202Z 2022-01-11T22:27:13.068515Z 2022-01-11T22:27:12.882404Z 2022-01-11T22:27:12.275162Z 2022-01-11T22:27:12.190303Z 2022-01-11T22:28:15.892925Z 2022-01-11T22:28:15.090965Z 2022-01-11T22:28:13.666313Z 2022-01-11T22:28:13.350365Z 2022-01-11T22:28:13.129236Z 2022-01-11T22:28:12.911422Z 2022-01-11T22:28:12.990640Z 2022-01-11T22:28:01.439712Z 2022-01-11T22:28:12.875217Z 2022-01-11T22:28:12.451204Z 2022-01-11T22:28:01.162995Z 2022-01-11T22:28:00.753386Z 2022-01-11T22:28:00.538996Z 2022-01-11T22:28:00.348987Z 2022-01-11T22:28:12.235561Z 2022-01-11T22:27:59.872864Z 2022-01-11T22:27:59.571879Z 2022-01-11T22:27:59.070647Z 2022-01-11T22:27:58.899122Z 2022-01-11T22:27:58.716722Z 2022-01-11T22:27:58.161485Z 2022-01-11T22:27:57.863550Z 2022-01-11T22:27:57.457666Z 2022-01-11T22:27:57.078710Z 2022-01-11T22:27:56.808478Z 2022-01-11T22:27:56.461271Z 2022-01-11T22:27:55.891768Z 2022-01-11T22:27:55.497469Z 2022-01-11T22:27:55.293447Z 2022-01-11T22:27:55.024109Z 2022-01-11T22:27:54.713086Z 2022-01-11T22:27:54.560834Z 2022-01-11T22:27:54.042710Z 2022-01-11T22:27:53.840255Z 2022-01-11T22:27:53.422287Z 2022-01-11T22:27:53.059217Z 2022-01-11T22:27:52.668406Z 2022-01-11T22:27:52.252177Z 2022-01-11T22:27:51.618165Z 2022-01-11T22:27:51.230317Z 2022-01-11T22:27:50.734346Z 2022-01-11T22:27:49.978255Z 2022-01-11T22:27:49.703399Z 2022-01-11T22:27:49.464304Z 2022-01-11T22:27:49.037681Z 2022-01-11T22:27:48.376040Z 2022-01-11T22:27:48.251589Z 2022-01-11T22:27:47.836819Z 2022-01-11T22:27:47.383275Z 2022-01-11T22:27:47.219512Z 2022-01-11T22:27:46.762950Z 2022-01-11T22:27:46.640512Z 2022-01-11T22:27:46.072984Z 2022-01-11T22:27:45.883498Z 2022-01-11T22:27:45.209398Z 2022-01-11T22:27:45.369116Z 2022-01-11T22:27:44.368141Z 2022-01-11T22:27:44.167115Z 2022-01-11T22:27:43.602226Z 2022-01-11T22:27:43.466602Z 2022-01-11T22:27:42.964156Z 2022-01-11T22:27:42.953875Z 2022-01-11T22:27:42.588470Z 2022-01-11T22:27:42.496408Z 2022-01-11T22:27:42.193101Z 2022-01-11T22:32:12.473370Z 2022-01-11T22:32:12.261263Z 2022-01-11T22:32:11.710724Z 2022-01-11T22:32:11.685505Z 2022-01-11T22:32:11.189248Z 2022-01-11T22:32:11.366522Z 2022-01-11T22:32:10.838488Z 2022-01-11T22:32:10.737432Z 2022-01-11T22:32:10.465167Z 2022-01-11T22:32:10.180564Z 2022-01-11T22:32:09.804501Z 2022-01-11T22:32:09.549730Z 2022-01-11T22:32:09.282439Z 2022-01-11T22:32:09.129340Z 2022-01-11T22:32:08.851611Z 2022-01-11T22:32:08.783423Z 2022-01-11T22:32:04.516074Z 2022-01-11T22:32:04.133511Z 2022-01-11T22:32:03.754800Z 2022-01-11T22:32:03.354479Z 2022-01-11T22:32:02.928073Z 2022-01-11T22:32:02.370485Z 2022-01-11T22:32:02.316824Z 2022-01-11T22:32:02.175363Z 2022-01-11T22:32:01.634471Z 2022-01-11T22:32:01.529033Z 2022-01-11T22:32:00.948943Z 2022-01-11T22:32:00.616977Z 2022-01-11T22:32:00.284323Z 2022-01-11T22:32:00.008663Z 2022-01-11T22:31:59.878215Z 2022-01-11T22:31:59.698355Z 2022-01-11T22:31:59.391854Z 2022-01-11T22:31:59.159022Z 2022-01-11T22:31:58.780315Z 2022-01-11T22:31:58.771429Z 2022-01-11T22:31:57.642066Z 2022-01-11T22:31:58.010479Z 2022-01-11T22:31:57.215272Z 2022-01-11T22:31:56.806419Z 2022-01-11T22:31:56.156944Z 2022-01-11T22:31:55.773659Z 2022-01-11T22:31:55.258329Z 2022-01-11T22:31:54.754694Z 2022-01-11T22:31:54.484193Z 2022-01-11T22:31:54.624795Z 2022-01-11T22:31:53.424231Z 2022-01-11T22:31:53.084755Z 2022-01-11T22:31:52.669446Z 2022-01-11T22:31:52.535381Z 2022-01-11T22:31:51.987300Z 2022-01-11T22:31:51.626045Z 2022-01-11T22:31:51.257249Z 2022-01-11T22:31:51.225699Z 2022-01-11T22:31:50.684220Z 2022-01-11T22:31:50.147490Z 2022-01-11T22:31:49.947377Z 2022-01-11T22:31:49.580622Z 2022-01-11T22:31:49.390030Z 2022-01-11T22:31:49.186386Z 2022-01-11T22:31:48.537515Z 2022-01-11T22:31:48.075388Z 2022-01-11T22:31:47.991398Z 2022-01-11T22:31:47.571295Z 2022-01-11T22:31:47.211356Z 2022-01-11T22:31:46.963519Z 2022-01-11T22:31:46.761064Z 2022-01-11T22:31:46.463809Z 2022-01-11T22:31:46.080574Z 2022-01-11T22:31:45.844510Z 2022-01-11T22:31:45.521845Z 2022-01-11T22:31:45.444473Z 2022-01-11T22:31:45.105507Z 2022-01-11T22:31:44.908049Z 2022-01-11T22:31:44.552233Z 2022-01-11T22:31:44.437811Z 2022-01-11T22:31:33.270946Z 2022-01-11T22:31:33.115325Z 2022-01-11T22:31:32.016127Z 2022-01-11T22:31:31.413705Z 2022-01-11T22:31:30.175646Z 2022-01-11T22:31:29.987971Z 2022-01-11T22:31:29.217773Z 2022-01-11T22:31:29.224630Z 2022-01-11T22:31:28.215063Z 2022-01-11T22:31:27.756666Z 2022-01-11T22:31:27.346493Z 2022-01-11T22:31:26.935045Z 2022-01-11T22:31:26.356748Z 2022-01-11T22:31:26.078407Z 2022-01-11T22:31:25.713696Z 2022-01-11T22:31:25.452572Z 2022-01-11T22:31:25.186170Z 2022-01-11T22:31:21.967639Z 2022-01-11T22:31:21.512071Z 2022-01-11T22:31:21.452854Z 2022-01-11T22:31:21.168637Z 2022-01-11T22:31:20.026948Z 2022-01-11T22:31:19.873711Z 2022-01-11T22:31:19.844760Z 2022-01-11T22:36:14.035683Z 2022-01-11T22:36:13.738607Z 2022-01-11T22:36:13.486267Z 2022-01-11T22:36:13.418893Z 2022-01-11T22:36:12.822600Z 2022-01-11T22:36:12.757820Z 2022-01-11T22:36:12.285987Z 2022-01-11T22:36:11.677583Z 2022-01-11T22:36:11.726686Z 2022-01-11T22:36:10.456056Z 2022-01-11T22:36:10.631821Z 2022-01-11T22:36:09.870314Z 2022-01-11T22:36:09.435961Z 2022-01-11T22:36:08.969757Z 2022-01-11T22:36:08.230177Z 2022-01-11T22:36:07.929997Z 2022-01-11T22:36:07.652113Z 2022-01-11T22:36:07.139284Z 2022-01-11T22:36:06.859483Z 2022-01-11T22:36:06.099274Z 2022-01-11T22:36:05.853688Z 2022-01-11T22:36:05.514373Z 2022-01-11T22:36:05.143621Z 2022-01-11T22:36:04.423275Z 2022-01-11T22:36:04.464863Z 2022-01-11T22:36:03.493505Z 2022-01-11T22:36:02.825378Z 2022-01-11T22:36:02.342332Z 2022-01-11T22:36:02.353114Z 2022-01-11T22:36:01.839182Z 2022-01-11T22:36:01.690668Z 2022-01-11T22:36:00.943305Z 2022-01-11T22:36:00.772178Z 2022-01-11T22:35:59.824296Z 2022-01-11T22:35:59.742630Z 2022-01-11T22:35:59.117306Z 2022-01-11T22:35:58.736943Z 2022-01-11T22:35:58.554291Z 2022-01-11T22:35:58.456757Z 2022-01-11T22:35:58.060121Z 2022-01-11T22:35:58.042589Z 2022-01-11T22:35:57.738791Z 2022-01-11T22:35:44.084374Z 2022-01-11T22:35:43.879407Z 2022-01-11T22:35:43.654602Z 2022-01-11T22:35:43.420240Z 2022-01-11T22:35:43.297070Z 2022-01-11T22:35:42.894475Z 2022-01-11T22:35:57.531393Z 2022-01-11T22:35:42.723080Z 2022-01-11T22:35:42.297406Z 2022-01-11T22:35:42.160351Z 2022-01-11T22:35:41.764236Z 2022-01-11T22:35:41.557393Z 2022-01-11T22:35:41.133357Z 2022-01-11T22:35:40.589087Z 2022-01-11T22:35:40.207352Z 2022-01-11T22:35:39.941810Z 2022-01-11T22:35:39.419291Z 2022-01-11T22:35:38.803441Z 2022-01-11T22:35:38.195131Z 2022-01-11T22:35:37.834872Z 2022-01-11T22:35:37.482952Z 2022-01-11T22:35:37.301199Z 2022-01-11T22:35:36.864543Z 2022-01-11T22:35:36.552970Z 2022-01-11T22:35:36.009010Z 2022-01-11T22:35:35.587832Z 2022-01-11T22:35:35.244417Z 2022-01-11T22:35:35.400495Z 2022-01-11T22:35:34.871728Z 2022-01-11T22:35:34.787245Z 2022-01-11T22:35:33.843300Z 2022-01-11T22:35:33.673309Z 2022-01-11T22:35:33.158242Z 2022-01-11T22:35:33.073629Z 2022-01-11T22:35:32.432191Z 2022-01-11T22:35:32.487902Z 2022-01-11T22:35:31.950484Z 2022-01-11T22:35:31.877411Z 2022-01-11T22:35:31.381485Z 2022-01-11T22:35:31.411843Z 2022-01-11T22:35:31.163747Z 2022-01-11T22:35:30.865127Z 2022-01-11T22:35:30.724725Z 2022-01-11T22:35:30.532329Z 2022-01-11T22:35:30.371781Z 2022-01-11T22:35:30.167391Z 2022-01-11T22:35:29.961514Z 2022-01-11T22:35:30.021269Z 2022-01-11T22:35:29.708064Z 2022-01-11T22:35:29.604377Z 2022-01-11T22:35:21.463893Z 2022-01-11T22:35:21.222062Z 2022-01-11T22:35:20.200502Z 2022-01-11T22:35:19.819430Z 2022-01-11T22:35:18.999217Z 2022-01-11T22:35:18.857234Z 2022-01-11T22:35:16.575632Z 2022-01-11T22:35:16.312067Z 2022-01-11T22:40:05.772337Z 2022-01-11T22:40:05.593421Z 2022-01-11T22:40:05.074455Z 2022-01-11T22:40:04.853086Z 2022-01-11T22:40:04.381300Z 2022-01-11T22:40:04.073315Z 2022-01-11T22:40:03.717211Z 2022-01-11T22:40:03.660995Z 2022-01-11T22:40:03.112124Z 2022-01-11T22:40:02.974225Z 2022-01-11T22:40:02.443382Z 2022-01-11T22:40:02.326290Z 2022-01-11T22:40:01.840352Z 2022-01-11T22:40:01.527361Z 2022-01-11T22:40:00.803453Z 2022-01-11T22:40:01.086542Z 2022-01-11T22:40:00.227255Z 2022-01-11T22:39:59.896097Z 2022-01-11T22:39:59.207216Z 2022-01-11T22:39:59.127363Z 2022-01-11T22:39:58.229364Z 2022-01-11T22:39:58.211829Z 2022-01-11T22:39:57.706181Z 2022-01-11T22:39:57.307348Z 2022-01-11T22:39:56.800292Z 2022-01-11T22:39:56.730935Z 2022-01-11T22:39:55.793912Z 2022-01-11T22:39:56.222308Z 2022-01-11T22:39:55.810178Z 2022-01-11T22:39:55.495452Z 2022-01-11T22:39:55.235454Z 2022-01-11T22:39:55.018775Z 2022-01-11T22:39:54.527651Z 2022-01-11T22:39:54.274373Z 2022-01-11T22:39:53.835365Z 2022-01-11T22:39:53.316500Z 2022-01-11T22:39:52.926410Z 2022-01-11T22:39:52.387698Z 2022-01-11T22:39:51.996510Z 2022-01-11T22:39:51.652524Z 2022-01-11T22:39:51.232481Z 2022-01-11T22:39:50.883787Z 2022-01-11T22:39:50.886521Z 2022-01-11T22:39:50.727774Z 2022-01-11T22:39:49.891077Z 2022-01-11T22:39:49.834783Z 2022-01-11T22:39:49.371021Z 2022-01-11T22:39:48.672288Z 2022-01-11T22:39:48.364064Z 2022-01-11T22:39:47.923438Z 2022-01-11T22:39:47.386414Z 2022-01-11T22:39:47.344173Z 2022-01-11T22:39:46.835540Z 2022-01-11T22:39:46.916842Z 2022-01-11T22:39:45.989222Z 2022-01-11T22:39:45.943699Z 2022-01-11T22:39:45.362623Z 2022-01-11T22:39:45.239388Z 2022-01-11T22:39:44.566446Z 2022-01-11T22:39:44.121388Z 2022-01-11T22:39:43.805971Z 2022-01-11T22:39:43.217745Z 2022-01-11T22:39:43.043538Z 2022-01-11T22:39:42.781417Z 2022-01-11T22:39:42.419438Z 2022-01-11T22:39:42.079183Z 2022-01-11T22:39:41.538586Z 2022-01-11T22:39:41.363908Z 2022-01-11T22:39:41.294339Z 2022-01-11T22:39:40.874948Z 2022-01-11T22:39:40.442006Z 2022-01-11T22:39:40.057651Z 2022-01-11T22:39:39.738895Z 2022-01-11T22:39:39.525230Z 2022-01-11T22:39:39.031471Z 2022-01-11T22:39:30.052538Z 2022-01-11T22:39:38.831511Z 2022-01-11T22:39:29.599235Z 2022-01-11T22:39:29.320779Z 2022-01-11T22:39:29.145310Z 2022-01-11T22:39:28.619595Z 2022-01-11T22:39:28.436302Z 2022-01-11T22:39:28.060393Z 2022-01-11T22:39:27.850005Z 2022-01-11T22:39:27.321834Z 2022-01-11T22:39:26.981346Z 2022-01-11T22:39:26.767363Z 2022-01-11T22:39:26.593228Z 2022-01-11T22:39:26.166043Z 2022-01-11T22:39:25.957494Z 2022-01-11T22:39:25.665349Z 2022-01-11T22:39:25.427705Z 2022-01-11T22:39:25.352452Z 2022-01-11T22:39:25.155968Z 2022-01-11T22:39:24.953380Z 2022-01-11T22:39:24.811927Z 2022-01-11T22:39:24.182224Z 2022-01-11T22:39:16.998005Z 2022-01-11T22:39:16.437378Z 2022-01-11T22:39:16.167726Z 2022-01-11T22:44:12.306246Z 2022-01-11T22:44:12.011001Z 2022-01-11T22:44:11.975734Z 2022-01-11T22:44:11.788404Z 2022-01-11T22:44:11.583057Z 2022-01-11T22:44:11.332523Z 2022-01-11T22:44:10.720751Z 2022-01-11T22:44:10.173938Z 2022-01-11T22:44:10.064421Z 2022-01-11T22:44:09.745262Z 2022-01-11T22:44:09.539357Z 2022-01-11T22:44:08.778494Z 2022-01-11T22:44:08.532445Z 2022-01-11T22:44:07.891200Z 2022-01-11T22:44:07.517705Z 2022-01-11T22:44:07.184382Z 2022-01-11T22:44:06.817268Z 2022-01-11T22:44:06.476225Z 2022-01-11T22:44:06.337494Z 2022-01-11T22:44:05.767205Z 2022-01-11T22:43:53.554077Z 2022-01-11T22:44:05.543842Z 2022-01-11T22:44:05.202058Z 2022-01-11T22:43:53.452930Z 2022-01-11T22:43:53.089173Z 2022-01-11T22:43:52.644586Z 2022-01-11T22:43:52.314954Z 2022-01-11T22:43:52.225200Z 2022-01-11T22:43:51.795753Z 2022-01-11T22:43:51.401565Z 2022-01-11T22:43:51.155130Z 2022-01-11T22:44:04.975305Z 2022-01-11T22:43:50.867426Z 2022-01-11T22:43:50.465246Z 2022-01-11T22:43:50.107211Z 2022-01-11T22:43:49.515393Z 2022-01-11T22:43:49.294207Z 2022-01-11T22:43:48.857031Z 2022-01-11T22:43:48.563270Z 2022-01-11T22:43:48.074175Z 2022-01-11T22:43:47.942398Z 2022-01-11T22:43:47.421204Z 2022-01-11T22:43:47.224302Z 2022-01-11T22:43:47.155096Z 2022-01-11T22:43:46.990118Z 2022-01-11T22:43:46.443025Z 2022-01-11T22:43:46.408287Z 2022-01-11T22:43:45.864080Z 2022-01-11T22:43:45.493402Z 2022-01-11T22:43:45.147697Z 2022-01-11T22:43:44.687236Z 2022-01-11T22:43:44.648390Z 2022-01-11T22:43:44.133033Z 2022-01-11T22:43:43.514214Z 2022-01-11T22:43:43.210391Z 2022-01-11T22:43:42.777854Z 2022-01-11T22:43:42.536269Z 2022-01-11T22:43:42.388354Z 2022-01-11T22:43:41.903891Z 2022-01-11T22:43:41.602997Z 2022-01-11T22:43:41.449687Z 2022-01-11T22:43:40.930743Z 2022-01-11T22:43:40.788402Z 2022-01-11T22:43:40.132506Z 2022-01-11T22:43:40.301179Z 2022-01-11T22:43:39.901506Z 2022-01-11T22:43:39.735093Z 2022-01-11T22:43:39.393322Z 2022-01-11T22:43:38.831606Z 2022-01-11T22:43:38.608273Z 2022-01-11T22:43:38.361248Z 2022-01-11T22:43:37.683999Z 2022-01-11T22:43:37.720989Z 2022-01-11T22:43:37.039292Z 2022-01-11T22:43:37.020064Z 2022-01-11T22:43:36.652174Z 2022-01-11T22:43:36.549372Z 2022-01-11T22:43:36.247474Z 2022-01-11T22:43:36.312611Z 2022-01-11T22:43:35.925603Z 2022-01-11T22:43:36.020217Z 2022-01-11T22:43:35.667245Z 2022-01-11T22:43:35.565515Z 2022-01-11T22:43:35.189726Z 2022-01-11T22:43:35.042740Z 2022-01-11T22:43:34.962391Z 2022-01-11T22:43:34.812597Z 2022-01-11T22:43:34.607177Z 2022-01-11T22:43:27.798728Z 2022-01-11T22:43:27.165178Z 2022-01-11T22:43:26.905497Z 2022-01-11T22:43:25.787518Z 2022-01-11T22:43:24.873845Z 2022-01-11T22:43:23.763400Z 2022-01-11T22:43:22.598248Z 2022-01-11T22:43:18.127947Z 2022-01-11T22:43:17.953453Z 2022-01-11T22:43:17.725333Z 2022-01-11T22:43:16.569878Z 2022-01-11T22:43:16.465414Z", + "started_diff": 55.900058200000004, + "successful_job_ids": " [99, 98, 97, 95, 94, 93, 92, 91, 90, 88, 87, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 72, 71, 70, 69, 68, 67, 66, 64, 63, 62, 61, 60, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 150, 149, 147, 146, 144, 143, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 171, 170, 169, 168, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 271, 270, 269, 268, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 371, 370, 369, 368, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, 471, 470, 469, 468]" + }, + "started": "2022-01-11T23:09:15.798353+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-12T03_04_04_712_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-12T03_04_04_712_0000-chatty.json new file mode 100644 index 00000000..9584b978 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-12T03_04_04_712_0000-chatty.json @@ -0,0 +1,12 @@ +{ + "ended": null, + "id": "run-2022-01-12T03_04_04_712_0000", + "measurements": {}, + "name": "This is default name", + "owner": "", + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-12T04:50:41.824106+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-api.json new file mode 100644 index 00000000..d925950e --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-api.json @@ -0,0 +1,10 @@ +{ + "ended": null, + "measurements": {}, + "name": null, + "owner": null, + "parameters": {}, + "result": null, + "results": {}, + "started": "2022-01-12T15:11:09.296728+00:00" +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-chatty.json new file mode 100644 index 00000000..7add9caa --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-chatty.json @@ -0,0 +1,637 @@ +{ + "ended": "2022-01-12T14:12:11.814778+00:00", + "golden": "true", + "id": "run-2022-01-12T11_58_55_803_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 7052.066666666666, + "max": 43167.73333333334, + "mean": 28925.334299516908, + "median": 33845.0, + "min": 1912.5333333333333, + "non_zero_mean": 28925.334299516908, + "non_zero_median": 33845.0, + "percentile25": 27486.066666666666, + "percentile75": 34538.13333333333, + "percentile90": 35767.33333333333, + "percentile99": 39265.62133333329, + "percentile999": 42777.52213333335, + "range": 41255.200000000004, + "samples": 69, + "stdev": 10103.085164032736, + "sum": 1995848.0666666667 + } + }, + "cpu": { + "system": { + "iqr": 0.23066666666666902, + "max": 1.269999999999999, + "mean": 1.0558260869565217, + "median": 1.225333333333337, + "min": 0.04133333333333553, + "non_zero_mean": 1.0558260869565217, + "non_zero_median": 1.225333333333337, + "percentile25": 1.0126666666666684, + "percentile75": 1.2433333333333374, + "percentile90": 1.261333333333326, + "percentile99": 1.2672800000000002, + "percentile999": 1.269727999999999, + "range": 1.2286666666666635, + "samples": 69, + "stdev": 0.3343746021129908, + "sum": 72.852 + }, + "user": { + "iqr": 1.9166666666666625, + "max": 6.401999999999995, + "mean": 5.129159420289855, + "median": 6.189333333333336, + "min": 0.19066666666666382, + "non_zero_mean": 5.129159420289855, + "non_zero_median": 6.189333333333336, + "percentile25": 4.313333333333341, + "percentile75": 6.230000000000003, + "percentile90": 6.25706666666668, + "percentile99": 6.397919999999998, + "percentile999": 6.4015919999999955, + "range": 6.211333333333331, + "samples": 69, + "stdev": 1.8071965276557223, + "sum": 353.912 + } + }, + "entropy_available_bits": { + "iqr": 7.0, + "max": 427.26666666666665, + "mean": 21.177777777777777, + "median": 2.7333333333333334, + "min": 0.0, + "non_zero_mean": 33.98294573643411, + "non_zero_median": 6.933333333333334, + "percentile25": 0.0, + "percentile75": 7.0, + "percentile90": 9.533333333333335, + "percentile99": 426.72266666666667, + "percentile999": 427.21226666666666, + "range": 427.26666666666665, + "samples": 69, + "stdev": 84.14655256971726, + "sum": 1461.2666666666667 + }, + "memory": { + "used": { + "iqr": 1304567808.0, + "max": 7650242560.0, + "mean": 6472279856.231884, + "median": 6824632320.0, + "min": 3492106240.0, + "non_zero_mean": 6472279856.231884, + "non_zero_median": 6824632320.0, + "percentile25": 5945233408.0, + "percentile75": 7249801216.0, + "percentile90": 7370443161.6, + "percentile99": 7622492815.36, + "percentile999": 7647467585.536, + "range": 4158136320.0, + "samples": 69, + "stdev": 986137702.9531157, + "sum": 446587310080.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.6, + "mean": 0.028985507246376812, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.6666666666666667, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.6933333333333243, + "percentile999": 1.5093333333333363, + "range": 1.6, + "samples": 69, + "stdev": 0.19519665326793387, + "sum": 2.0 + } + }, + "writes_completed": { + "total": { + "iqr": 1999.8666666666663, + "max": 4547.466666666667, + "mean": 3309.9217391304346, + "median": 4274.4, + "min": 2.8000000000000003, + "non_zero_mean": 3309.9217391304346, + "non_zero_median": 4274.4, + "percentile25": 2364.3333333333335, + "percentile75": 4364.2, + "percentile90": 4444.333333333333, + "percentile99": 4537.085333333333, + "percentile999": 4546.428533333334, + "range": 4544.666666666667, + "samples": 69, + "stdev": 1520.08368951567, + "sum": 228384.59999999992 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 203.4000000000001, + "max": 2016.6666666666667, + "mean": 1518.791304347826, + "median": 1566.6, + "min": 494.8, + "non_zero_mean": 1518.791304347826, + "non_zero_median": 1566.6, + "percentile25": 1480.7333333333333, + "percentile75": 1684.1333333333334, + "percentile90": 1748.8, + "percentile99": 2012.8133333333333, + "percentile999": 2016.2813333333334, + "range": 1521.8666666666668, + "samples": 69, + "stdev": 298.96516441069986, + "sum": 104796.60000000003 + } + }, + "cpu": { + "system": { + "iqr": 0.029333333333333184, + "max": 0.20933333333333623, + "mean": 0.14837681159420293, + "median": 0.1580000000000003, + "min": 0.02000000000000076, + "non_zero_mean": 0.14837681159420293, + "non_zero_median": 0.1580000000000003, + "percentile25": 0.14066666666666758, + "percentile75": 0.17000000000000076, + "percentile90": 0.18706666666666633, + "percentile99": 0.20480000000000181, + "percentile999": 0.2088800000000028, + "range": 0.18933333333333546, + "samples": 69, + "stdev": 0.04091653839856413, + "sum": 10.238000000000001 + }, + "user": { + "iqr": 0.05933333333333196, + "max": 0.5840000000000003, + "mean": 0.23207729468599037, + "median": 0.1726666666666669, + "min": 0.023999999999999962, + "non_zero_mean": 0.23207729468599037, + "non_zero_median": 0.1726666666666669, + "percentile25": 0.16333333333333305, + "percentile75": 0.22266666666666501, + "percentile90": 0.5245333333333326, + "percentile99": 0.5694933333333332, + "percentile999": 0.5825493333333337, + "range": 0.5600000000000004, + "samples": 69, + "stdev": 0.1475098107397237, + "sum": 16.013333333333335 + } + }, + "entropy_available_bits": { + "iqr": 0.2666666666666666, + "max": 196.93333333333334, + "mean": 9.419323671497585, + "median": 1.2, + "min": 0.26666666666666666, + "non_zero_mean": 9.419323671497585, + "non_zero_median": 1.2, + "percentile25": 1.1333333333333333, + "percentile75": 1.4, + "percentile90": 1.6666666666666667, + "percentile99": 191.81066666666663, + "percentile999": 196.4210666666667, + "range": 196.66666666666666, + "samples": 69, + "stdev": 38.73105541789713, + "sum": 649.9333333333332 + }, + "memory": { + "used": { + "iqr": 26382336.0, + "max": 482967552.0, + "mean": 449046498.31884056, + "median": 461717504.0, + "min": 372236288.0, + "non_zero_mean": 449046498.31884056, + "non_zero_median": 461717504.0, + "percentile25": 441982976.0, + "percentile75": 468365312.0, + "percentile90": 474755891.2, + "percentile99": 482597109.76, + "percentile999": 482930507.776, + "range": 110731264.0, + "samples": 69, + "stdev": 32142252.143688794, + "sum": 30984208384.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1796778.6666666667, + "max": 3217271.466666667, + "mean": 762125.109178744, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1947653.0567901235, + "non_zero_median": 1907097.6, + "percentile25": 0.0, + "percentile75": 1796778.6666666667, + "percentile90": 2791396.6933333334, + "percentile99": 3059067.562666665, + "percentile999": 3201451.0762666673, + "range": 3217271.466666667, + "samples": 69, + "stdev": 1107610.830882373, + "sum": 52586632.53333334 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.293623, + "mean": 0.006771086956521739, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.035938846153846155, + "non_zero_median": 0.013491, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.010115000000000011, + "percentile99": 0.12042019999999826, + "percentile999": 0.27630272000000056, + "range": 0.293623, + "samples": 69, + "stdev": 0.03572518084947397, + "sum": 0.46720500000000004 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.0, + "max": 0.779539, + "mean": 0.03740679710144928, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.1613168125, + "non_zero_median": 0.099924, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.10876280000000005, + "percentile99": 0.6656498799999988, + "percentile999": 0.7681500880000004, + "range": 0.779539, + "samples": 69, + "stdev": 0.12758110270317555, + "sum": 2.581069 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 9.2, + "mean": 0.27342995169082124, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6.288888888888889, + "non_zero_median": 5.466666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 6.661333333333308, + "percentile999": 8.946133333333341, + "range": 9.2, + "samples": 69, + "stdev": 1.3666496855729457, + "sum": 18.866666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9995.466666666667, + "mean": 434.3478260869565, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9990.0, + "non_zero_median": 9992.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9993.109333333334, + "percentile999": 9995.230933333334, + "range": 9995.466666666667, + "samples": 69, + "stdev": 2052.1974781987806, + "sum": 29970.0 + }, + "pg_stat_database_blks_hit": { + "iqr": 2389.1333333333314, + "max": 31768.333333333332, + "mean": 19808.422222222223, + "median": 20259.466666666667, + "min": 3150.5333333333333, + "non_zero_mean": 19808.422222222223, + "non_zero_median": 20259.466666666667, + "percentile25": 19006.466666666667, + "percentile75": 21395.6, + "percentile90": 25757.160000000003, + "percentile99": 31575.031999999996, + "percentile999": 31749.0032, + "range": 28617.8, + "samples": 69, + "stdev": 5265.320640053254, + "sum": 1366781.1333333335 + }, + "pg_stat_database_blks_read": { + "iqr": 46.66666666666666, + "max": 162.93333333333334, + "mean": 92.96231884057971, + "median": 110.73333333333333, + "min": 0.0, + "non_zero_mean": 94.32941176470588, + "non_zero_median": 110.8, + "percentile25": 72.4, + "percentile75": 119.06666666666666, + "percentile90": 125.89333333333333, + "percentile99": 162.66133333333332, + "percentile999": 162.90613333333334, + "range": 162.93333333333334, + "samples": 69, + "stdev": 43.11972916090635, + "sum": 6414.4 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 69, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 46.66666666666666, + "max": 162.93333333333334, + "mean": 92.96231884057971, + "median": 110.73333333333333, + "min": 0.0, + "non_zero_mean": 94.32941176470588, + "non_zero_median": 110.8, + "percentile25": 72.4, + "percentile75": 119.06666666666666, + "percentile90": 125.89333333333333, + "percentile99": 162.66133333333332, + "percentile999": 162.90613333333334, + "range": 162.93333333333334, + "samples": 69, + "stdev": 43.11972916090635, + "sum": 6414.4 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.3333333333333333, + "mean": 0.19903381642512077, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.9155555555555556, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.2879999999999996, + "percentile999": 1.3288, + "range": 1.3333333333333333, + "samples": 69, + "stdev": 0.4062409783610747, + "sum": 13.733333333333334 + }, + "pg_stat_database_tup_fetched": { + "iqr": 3504.9333333333334, + "max": 18632.4, + "mean": 6418.884057971014, + "median": 4588.133333333333, + "min": 1645.3333333333333, + "non_zero_mean": 6418.884057971014, + "non_zero_median": 4588.133333333333, + "percentile25": 4149.8, + "percentile75": 7654.733333333334, + "percentile90": 12177.386666666667, + "percentile99": 17765.89866666666, + "percentile999": 18545.749866666672, + "range": 16987.06666666667, + "samples": 69, + "stdev": 3668.454144640094, + "sum": 442903.0000000001 + }, + "pg_stat_database_tup_inserted": { + "iqr": 255.73333333333335, + "max": 902.9333333333333, + "mean": 495.056038647343, + "median": 579.4, + "min": 0.0, + "non_zero_mean": 502.3362745098039, + "non_zero_median": 579.4333333333334, + "percentile25": 379.73333333333335, + "percentile75": 635.4666666666667, + "percentile90": 671.5733333333333, + "percentile99": 874.8266666666664, + "percentile999": 900.1226666666668, + "range": 902.9333333333333, + "samples": 69, + "stdev": 229.89675847777133, + "sum": 34158.86666666667 + }, + "pg_stat_database_tup_returned": { + "iqr": 4613.666666666666, + "max": 24580.733333333334, + "mean": 8241.504347826087, + "median": 6021.133333333333, + "min": 2594.866666666667, + "non_zero_mean": 8241.504347826087, + "non_zero_median": 6021.133333333333, + "percentile25": 5317.133333333333, + "percentile75": 9930.8, + "percentile90": 15821.066666666671, + "percentile99": 21934.445333333308, + "percentile999": 24316.104533333342, + "range": 21985.86666666667, + "samples": 69, + "stdev": 4779.970973008896, + "sum": 568663.8000000002 + }, + "pg_stat_database_tup_updated": { + "iqr": 5.733333333333333, + "max": 96.93333333333334, + "mean": 9.858937198067633, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 11.151912568306011, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.26666666666666666, + "percentile75": 6.0, + "percentile90": 29.80000000000004, + "percentile99": 96.75200000000001, + "percentile999": 96.9152, + "range": 96.93333333333334, + "samples": 69, + "stdev": 22.107028829999802, + "sum": 680.2666666666667 + }, + "pg_stat_database_xact_commit": { + "iqr": 10.666666666666671, + "max": 147.06666666666666, + "mean": 52.30338164251208, + "median": 44.8, + "min": 8.666666666666666, + "non_zero_mean": 52.30338164251208, + "non_zero_median": 44.8, + "percentile25": 40.8, + "percentile75": 51.46666666666667, + "percentile90": 82.49333333333335, + "percentile99": 146.15999999999997, + "percentile999": 146.976, + "range": 138.4, + "samples": 69, + "stdev": 27.027769636682557, + "sum": 3608.9333333333334 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 69, + "stdev": 0.0, + "sum": 4.600000000000003 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.4, + "mean": 0.005797101449275362, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.4, + "non_zero_median": 0.4, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.12799999999999728, + "percentile999": 0.3728000000000009, + "range": 0.4, + "samples": 69, + "stdev": 0.04815434123430768, + "sum": 0.4 + } + }, + "writes_completed": { + "total": { + "iqr": 19.46666666666667, + "max": 133.39999999999998, + "mean": 66.72367149758453, + "median": 66.53333333333333, + "min": 5.0, + "non_zero_mean": 66.72367149758453, + "non_zero_median": 66.53333333333333, + "percentile25": 57.2, + "percentile75": 76.66666666666667, + "percentile90": 96.02666666666667, + "percentile99": 127.95999999999994, + "percentile999": 132.856, + "range": 128.39999999999998, + "samples": 69, + "stdev": 23.441121555632186, + "sum": 4603.933333333335 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "5,4,3,6,7,9,8,10,11,14,18,19,21,22,20,23,24,25,26,27,28,29,30,34,31,33,32,35,36,37,39,38,40,41,43,42,44,45,46,47,49,48,50,52,51,54,53,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-12T13:52:17.954658Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2022-01-12T13:52:19.142944Z 2022-01-12T13:52:19.031032Z 2022-01-12T13:52:18.612234Z 2022-01-12T13:52:18.012139Z 2022-01-12T13:52:18.009264Z 2022-01-12T13:52:17.981903Z 2022-01-12T13:52:17.954658Z 2022-01-12T13:52:19.840785Z 2022-01-12T13:52:19.209802Z 2022-01-12T13:52:19.170900Z 2022-01-12T13:56:41.742045Z 2022-01-12T13:56:41.701163Z 2022-01-12T13:56:41.511568Z 2022-01-12T13:56:41.437036Z 2022-01-12T13:56:40.949836Z 2022-01-12T13:56:40.879773Z 2022-01-12T13:56:40.867253Z 2022-01-12T13:56:40.796570Z 2022-01-12T13:56:40.795980Z 2022-01-12T13:56:40.794283Z 2022-01-12T13:59:55.231540Z 2022-01-12T13:59:55.166239Z 2022-01-12T13:59:54.674390Z 2022-01-12T13:59:54.612309Z 2022-01-12T13:59:54.603622Z 2022-01-12T13:59:54.585493Z 2022-01-12T13:59:54.561878Z 2022-01-12T13:59:54.525068Z 2022-01-12T13:59:54.494163Z 2022-01-12T13:59:54.493324Z 2022-01-12T14:03:11.217197Z 2022-01-12T14:03:11.203771Z 2022-01-12T14:03:10.800571Z 2022-01-12T14:03:10.746127Z 2022-01-12T14:03:10.736556Z 2022-01-12T14:03:10.727324Z 2022-01-12T14:03:10.663964Z 2022-01-12T14:03:10.658579Z 2022-01-12T14:03:10.585235Z 2022-01-12T14:03:10.582350Z 2022-01-12T14:06:29.017603Z 2022-01-12T14:06:29.010085Z 2022-01-12T14:06:28.571242Z 2022-01-12T14:06:28.546445Z 2022-01-12T14:06:28.537652Z 2022-01-12T14:06:28.520469Z 2022-01-12T14:06:28.496248Z 2022-01-12T14:06:28.487677Z 2022-01-12T14:06:28.398969Z 2022-01-12T14:06:28.383355Z", + "created_diff": 0.9682400000000001, + "ended": "2022-01-12T14:09:22.914740Z", + "error_job_ids": " []", + "event_first": "2022-01-12T13:52:33.924245Z 2022-01-12T13:53:20.941295Z 2022-01-12T13:52:29.517366Z 2022-01-12T13:53:34.807198Z 2022-01-12T13:52:24.764717Z 2022-01-12T13:53:10.945463Z 2022-01-12T13:52:24.321631Z 2022-01-12T13:53:48.466081Z 2022-01-12T13:52:34.366144Z 2022-01-12T13:54:09.849486Z 2022-01-12T13:56:56.869558Z 2022-01-12T13:56:56.271100Z 2022-01-12T13:56:53.731114Z 2022-01-12T13:56:54.493156Z 2022-01-12T13:56:49.094867Z 2022-01-12T13:56:49.273150Z 2022-01-12T13:56:48.587162Z 2022-01-12T13:56:49.795101Z 2022-01-12T13:56:48.090144Z 2022-01-12T13:56:47.733017Z 2022-01-12T14:00:12.913025Z 2022-01-12T14:00:14.070369Z 2022-01-12T14:00:02.633115Z 2022-01-12T14:00:07.975920Z 2022-01-12T14:00:02.635386Z 2022-01-12T14:00:06.061285Z 2022-01-12T14:00:02.632987Z 2022-01-12T14:00:01.498201Z 2022-01-12T14:00:02.131238Z 2022-01-12T14:00:01.315292Z 2022-01-12T14:03:34.345262Z 2022-01-12T14:03:34.918520Z 2022-01-12T14:03:22.144511Z 2022-01-12T14:03:21.433695Z 2022-01-12T14:03:18.650919Z 2022-01-12T14:03:18.465444Z 2022-01-12T14:03:17.781147Z 2022-01-12T14:03:17.434543Z 2022-01-12T14:03:17.014899Z 2022-01-12T14:03:17.227279Z 2022-01-12T14:06:52.205071Z 2022-01-12T14:06:53.942384Z 2022-01-12T14:06:38.560330Z 2022-01-12T14:06:38.579422Z 2022-01-12T14:06:36.841442Z 2022-01-12T14:06:37.951179Z 2022-01-12T14:06:35.272137Z 2022-01-12T14:06:35.234186Z 2022-01-12T14:06:34.777829Z 2022-01-12T14:06:34.997423Z", + "event_last": "2022-01-12T13:55:18.905695Z 2022-01-12T13:55:53.564637Z 2022-01-12T13:55:12.177662Z 2022-01-12T13:56:04.372117Z 2022-01-12T13:55:10.669525Z 2022-01-12T13:55:47.481011Z 2022-01-12T13:55:10.517876Z 2022-01-12T13:56:13.064312Z 2022-01-12T13:55:18.323655Z 2022-01-12T13:56:25.825176Z 2022-01-12T13:59:29.019587Z 2022-01-12T13:59:33.765466Z 2022-01-12T13:59:28.134337Z 2022-01-12T13:59:32.038013Z 2022-01-12T13:59:25.179471Z 2022-01-12T13:59:30.500603Z 2022-01-12T13:59:24.472322Z 2022-01-12T13:59:31.507318Z 2022-01-12T13:59:24.633648Z 2022-01-12T13:59:30.992524Z 2022-01-12T14:02:42.735329Z 2022-01-12T14:02:46.771981Z 2022-01-12T14:02:36.733940Z 2022-01-12T14:02:44.551981Z 2022-01-12T14:02:35.335252Z 2022-01-12T14:02:43.812290Z 2022-01-12T14:02:36.279090Z 2022-01-12T14:02:42.702806Z 2022-01-12T14:02:35.735156Z 2022-01-12T14:02:42.843367Z 2022-01-12T14:06:05.042270Z 2022-01-12T14:06:06.362697Z 2022-01-12T14:05:55.302797Z 2022-01-12T14:06:02.732729Z 2022-01-12T14:05:53.947437Z 2022-01-12T14:06:03.198209Z 2022-01-12T14:05:53.234664Z 2022-01-12T14:06:02.732729Z 2022-01-12T14:05:52.723824Z 2022-01-12T14:06:01.863070Z 2022-01-12T14:09:22.914740Z 2022-01-12T14:09:22.595264Z 2022-01-12T14:09:10.448847Z 2022-01-12T14:09:20.171984Z 2022-01-12T14:09:10.240770Z 2022-01-12T14:09:19.877398Z 2022-01-12T14:09:09.518954Z 2022-01-12T14:09:19.156505Z 2022-01-12T14:09:09.560057Z 2022-01-12T14:09:18.336809Z", + "failed_job_ids": " []", + "finished": "2022-01-12T13:55:22.772986Z 2022-01-12T13:55:57.770140Z 2022-01-12T13:55:16.328867Z 2022-01-12T13:56:07.941536Z 2022-01-12T13:55:12.221384Z 2022-01-12T13:55:52.223136Z 2022-01-12T13:55:11.131647Z 2022-01-12T13:56:16.339490Z 2022-01-12T13:55:22.393898Z 2022-01-12T13:56:27.470492Z 2022-01-12T13:59:32.315515Z 2022-01-12T13:59:33.531981Z 2022-01-12T13:59:32.199535Z 2022-01-12T13:59:32.968884Z 2022-01-12T13:59:28.576957Z 2022-01-12T13:59:27.577977Z 2022-01-12T13:59:27.276770Z 2022-01-12T13:59:30.525555Z 2022-01-12T13:59:27.287078Z 2022-01-12T13:59:29.065118Z 2022-01-12T14:02:46.271081Z 2022-01-12T14:02:48.602232Z 2022-01-12T14:02:40.702523Z 2022-01-12T14:02:46.035635Z 2022-01-12T14:02:38.331108Z 2022-01-12T14:02:43.621141Z 2022-01-12T14:02:39.166565Z 2022-01-12T14:02:40.167334Z 2022-01-12T14:02:38.955312Z 2022-01-12T14:02:40.799299Z 2022-01-12T14:06:07.354807Z 2022-01-12T14:06:08.110430Z 2022-01-12T14:05:59.697531Z 2022-01-12T14:05:59.886546Z 2022-01-12T14:05:57.466250Z 2022-01-12T14:06:01.520411Z 2022-01-12T14:05:56.330129Z 2022-01-12T14:06:00.201483Z 2022-01-12T14:05:55.918559Z 2022-01-12T14:06:00.064574Z 2022-01-12T14:09:24.885138Z 2022-01-12T14:09:26.022815Z 2022-01-12T14:09:14.672078Z 2022-01-12T14:09:19.795341Z 2022-01-12T14:09:14.194224Z 2022-01-12T14:09:17.255215Z 2022-01-12T14:09:13.026213Z 2022-01-12T14:09:16.351714Z 2022-01-12T14:09:12.512763Z 2022-01-12T14:09:14.760723Z", + "finished_diff": 23.713420600000003, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 246.431949, + "mean": 173.27159837999997, + "min": 162.27834 + }, + "jobs_events_duration": { + "max": 166.196245, + "mean": 156.70242932, + "min": 135.97569 + }, + "jobs_events_lag": { + "max": 3.576086, + "mean": -1.5598043799999999, + "min": -4.742125 + }, + "jobs_waiting": { + "max": 14.235859, + "mean": 2.5547517600000003, + "min": 0.922868 + }, + "modified": "2022-01-12T13:52:20.502157Z 2022-01-12T13:52:20.468637Z 2022-01-12T13:52:20.410919Z 2022-01-12T13:52:18.654182Z 2022-01-12T13:52:18.619616Z 2022-01-12T13:52:18.584053Z 2022-01-12T13:52:18.544809Z 2022-01-12T13:52:33.848610Z 2022-01-12T13:52:20.579326Z 2022-01-12T13:52:20.543353Z 2022-01-12T13:56:42.999027Z 2022-01-12T13:56:42.948399Z 2022-01-12T13:56:42.800083Z 2022-01-12T13:56:42.707235Z 2022-01-12T13:56:41.568151Z 2022-01-12T13:56:41.527649Z 2022-01-12T13:56:41.493150Z 2022-01-12T13:56:41.460376Z 2022-01-12T13:56:41.427375Z 2022-01-12T13:56:41.392545Z 2022-01-12T13:59:57.521830Z 2022-01-12T13:59:57.461908Z 2022-01-12T13:59:55.389514Z 2022-01-12T13:59:55.355141Z 2022-01-12T13:59:55.319661Z 2022-01-12T13:59:55.276210Z 2022-01-12T13:59:55.232905Z 2022-01-12T13:59:55.198182Z 2022-01-12T13:59:55.162635Z 2022-01-12T13:59:55.126345Z 2022-01-12T14:03:20.271360Z 2022-01-12T14:03:20.041134Z 2022-01-12T14:03:11.444566Z 2022-01-12T14:03:11.411813Z 2022-01-12T14:03:11.378180Z 2022-01-12T14:03:11.345256Z 2022-01-12T14:03:11.307186Z 2022-01-12T14:03:11.272707Z 2022-01-12T14:03:11.238531Z 2022-01-12T14:03:11.202230Z 2022-01-12T14:06:41.658186Z 2022-01-12T14:06:41.581567Z 2022-01-12T14:06:29.231685Z 2022-01-12T14:06:29.198975Z 2022-01-12T14:06:29.167002Z 2022-01-12T14:06:29.133448Z 2022-01-12T14:06:29.095098Z 2022-01-12T14:06:29.062265Z 2022-01-12T14:06:29.029622Z 2022-01-12T14:06:28.995548Z", + "modified_diff": 8.2075072, + "started": "2022-01-12T13:52:20.939303Z 2022-01-12T13:52:20.861435Z 2022-01-12T13:52:20.795597Z 2022-01-12T13:52:19.171389Z 2022-01-12T13:52:19.050428Z 2022-01-12T13:52:19.010472Z 2022-01-12T13:52:18.877526Z 2022-01-12T13:52:34.076644Z 2022-01-12T13:52:21.212954Z 2022-01-12T13:52:21.038543Z 2022-01-12T13:56:43.699588Z 2022-01-12T13:56:43.572854Z 2022-01-12T13:56:43.406799Z 2022-01-12T13:56:43.251617Z 2022-01-12T13:56:42.266422Z 2022-01-12T13:56:42.182550Z 2022-01-12T13:56:42.046925Z 2022-01-12T13:56:41.967619Z 2022-01-12T13:56:41.876500Z 2022-01-12T13:56:41.782402Z 2022-01-12T13:59:58.000707Z 2022-01-12T13:59:57.877014Z 2022-01-12T13:59:56.325016Z 2022-01-12T13:59:56.268305Z 2022-01-12T13:59:56.052768Z 2022-01-12T13:59:55.945487Z 2022-01-12T13:59:55.849479Z 2022-01-12T13:59:55.748606Z 2022-01-12T13:59:55.665919Z 2022-01-12T13:59:55.573456Z 2022-01-12T14:03:21.690405Z 2022-01-12T14:03:21.503726Z 2022-01-12T14:03:12.413506Z 2022-01-12T14:03:12.307539Z 2022-01-12T14:03:12.181961Z 2022-01-12T14:03:12.095410Z 2022-01-12T14:03:12.017525Z 2022-01-12T14:03:11.927587Z 2022-01-12T14:03:11.839702Z 2022-01-12T14:03:11.764279Z 2022-01-12T14:06:42.129444Z 2022-01-12T14:06:42.241663Z 2022-01-12T14:06:30.147701Z 2022-01-12T14:06:30.046625Z 2022-01-12T14:06:29.931607Z 2022-01-12T14:06:29.844101Z 2022-01-12T14:06:29.758316Z 2022-01-12T14:06:29.684312Z 2022-01-12T14:06:29.589180Z 2022-01-12T14:06:29.509288Z", + "started_diff": 8.4404112, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-12T14:12:09.338518+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-launching.json new file mode 100644 index 00000000..dae69f73 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-12T11_58_55_803_0000-launching.json @@ -0,0 +1,635 @@ +{ + "ended": "2022-01-12T15:10:19.297878+00:00", + "golden": "true", + "id": "run-2022-01-12T11_58_55_803_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 14097.633333333335, + "max": 28778.466666666667, + "mean": 7880.332235531235, + "median": 1933.3385606969819, + "min": 1162.8666666666668, + "non_zero_mean": 7880.332235531235, + "non_zero_median": 1933.3385606969819, + "percentile25": 1616.5333333333333, + "percentile75": 15714.166666666668, + "percentile90": 20826.773333333334, + "percentile99": 27142.467999999997, + "percentile999": 28589.399200000033, + "range": 27615.6, + "samples": 119, + "stdev": 8402.109248691215, + "sum": 937759.5360282167 + } + }, + "cpu": { + "system": { + "iqr": 1.5529999999999984, + "max": 2.3006666666666584, + "mean": 0.6755293998268034, + "median": 0.05533333333333227, + "min": 0.02933333333333318, + "non_zero_mean": 0.6755293998268034, + "non_zero_median": 0.05533333333333227, + "percentile25": 0.04000000000000152, + "percentile75": 1.593, + "percentile90": 1.8889333333333276, + "percentile99": 2.22936, + "percentile999": 2.2932719999999938, + "range": 2.271333333333325, + "samples": 119, + "stdev": 0.8166676643370616, + "sum": 80.38799857938962 + }, + "user": { + "iqr": 5.465666666666659, + "max": 6.233999999999954, + "mean": 2.219064301100278, + "median": 0.38468604702847636, + "min": 0.0740000000000009, + "non_zero_mean": 2.219064301100278, + "non_zero_median": 0.38468604702847636, + "percentile25": 0.12900000000000017, + "percentile75": 5.59466666666666, + "percentile90": 5.935600000000017, + "percentile99": 6.206679999999992, + "percentile999": 6.230931999999959, + "range": 6.159999999999953, + "samples": 119, + "stdev": 2.5518377822179215, + "sum": 264.06865183093305 + } + }, + "entropy_available_bits": { + "iqr": 3.5, + "max": 424.0, + "mean": 22.599662999130018, + "median": 0.8, + "min": 0.0, + "non_zero_mean": 22.791185566919257, + "non_zero_median": 0.8, + "percentile25": 0.5666666666666667, + "percentile75": 4.066666666666666, + "percentile90": 5.08, + "percentile99": 420.3853333333333, + "percentile999": 423.6853333333334, + "range": 424.0, + "samples": 119, + "stdev": 89.99238107967649, + "sum": 2689.359896896473 + }, + "memory": { + "used": { + "iqr": 4914182144.0, + "max": 17835188224.0, + "mean": 8003661101.176471, + "median": 6235398144.0, + "min": 4816314368.0, + "non_zero_mean": 8003661101.176471, + "non_zero_median": 6235398144.0, + "percentile25": 5214754816.0, + "percentile75": 10128936960.0, + "percentile90": 14894535475.2, + "percentile99": 17497756794.879993, + "percentile999": 17819192000.512, + "range": 13018873856.0, + "samples": 119, + "stdev": 3776369218.0041313, + "sum": 952435671040.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.7333333333333334, + "mean": 0.01680672268907563, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.0, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.21866666666666484, + "percentile999": 1.560266666666695, + "range": 1.7333333333333334, + "samples": 119, + "stdev": 0.1605589313156769, + "sum": 2.0 + } + }, + "writes_completed": { + "total": { + "iqr": 618.3333333333334, + "max": 1567.8, + "mean": 333.59383835740306, + "median": 7.06693336889363, + "min": 0.0, + "non_zero_mean": 342.22126521147385, + "non_zero_median": 8.766666666666667, + "percentile25": 1.3666666666666667, + "percentile75": 619.7, + "percentile90": 1199.3066666666668, + "percentile99": 1503.7973333333332, + "percentile999": 1560.2873333333346, + "range": 1567.8, + "samples": 119, + "stdev": 489.2074676105541, + "sum": 39697.66676453096 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 1961.7666666666664, + "max": 3778.733333333333, + "mean": 1390.1254901960783, + "median": 741.5333333333333, + "min": 366.8, + "non_zero_mean": 1390.1254901960783, + "non_zero_median": 741.5333333333333, + "percentile25": 584.4333333333334, + "percentile75": 2546.2, + "percentile90": 2832.7866666666664, + "percentile99": 3307.747999999999, + "percentile999": 3724.7444000000087, + "range": 3411.933333333333, + "samples": 119, + "stdev": 1021.8920675286979, + "sum": 165424.9333333334 + } + }, + "cpu": { + "system": { + "iqr": 0.21566666666666665, + "max": 0.42399999999999904, + "mean": 0.11327170868347339, + "median": 0.03266666666666917, + "min": 0.010666666666668333, + "non_zero_mean": 0.11327170868347339, + "non_zero_median": 0.03266666666666917, + "percentile25": 0.02133333333333193, + "percentile75": 0.23699999999999857, + "percentile90": 0.2994666666666626, + "percentile99": 0.3772533333333315, + "percentile999": 0.4189653333333331, + "range": 0.4133333333333307, + "samples": 119, + "stdev": 0.12055552491818282, + "sum": 13.479333333333331 + }, + "user": { + "iqr": 0.25033333333333163, + "max": 0.773333333333333, + "mean": 0.14870028011204484, + "median": 0.046666666666665906, + "min": 0.010666666666667386, + "non_zero_mean": 0.14870028011204484, + "non_zero_median": 0.046666666666665906, + "percentile25": 0.018333333333334187, + "percentile75": 0.26866666666666583, + "percentile90": 0.3814666666666688, + "percentile99": 0.6321733333333333, + "percentile999": 0.7572853333333357, + "range": 0.7626666666666656, + "samples": 119, + "stdev": 0.17105483859261983, + "sum": 17.69533333333334 + } + }, + "entropy_available_bits": { + "iqr": 2.9, + "max": 212.4, + "mean": 11.523249299719888, + "median": 0.3333333333333333, + "min": 0.0, + "non_zero_mean": 15.761685823754789, + "non_zero_median": 0.4666666666666667, + "percentile25": 0.0, + "percentile75": 2.9, + "percentile90": 3.96, + "percentile99": 210.66666666666666, + "percentile999": 212.1954666666667, + "range": 212.4, + "samples": 119, + "stdev": 45.645530709169016, + "sum": 1371.2666666666664 + }, + "memory": { + "used": { + "iqr": 284045312.0, + "max": 1200934912.0, + "mean": 541120873.4117647, + "median": 380649472.0, + "min": 365137920.0, + "non_zero_mean": 541120873.4117647, + "non_zero_median": 380649472.0, + "percentile25": 373426176.0, + "percentile75": 657471488.0, + "percentile90": 997027840.0, + "percentile99": 1134936719.36, + "percentile999": 1193511477.248001, + "range": 835796992.0, + "samples": 119, + "stdev": 251000586.6259291, + "sum": 64393383936.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 0.0, + "max": 141448.53333333333, + "mean": 11060.347338935575, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 54840.88888888889, + "non_zero_median": 49425.066666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 48933.546666666676, + "percentile99": 136391.33866666665, + "percentile999": 140932.98346666674, + "range": 141448.53333333333, + "samples": 119, + "stdev": 29235.643497534707, + "sum": 1316181.3333333335 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 2.025792, + "mean": 0.03522718487394958, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.34933625, + "non_zero_median": 0.04292, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0001352000000000019, + "percentile99": 1.1081056399999962, + "percentile999": 1.9293304220000158, + "range": 2.025792, + "samples": 119, + "stdev": 0.22300927889709615, + "sum": 4.192035 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.12046000000000001, + "max": 11.622786, + "mean": 0.5600168739495798, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.5498141395348837, + "non_zero_median": 0.6103, + "percentile25": 0.0, + "percentile75": 0.12046000000000001, + "percentile90": 1.4131784, + "percentile99": 6.501675979999994, + "percentile999": 11.037472724000095, + "range": 11.622786, + "samples": 119, + "stdev": 1.5835909893669233, + "sum": 66.64200799999999 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 3.1333333333333333, + "mean": 0.08291316526610644, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.6444444444444444, + "non_zero_median": 1.6666666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.6666666666666667, + "percentile999": 2.960266666666695, + "range": 3.1333333333333333, + "samples": 119, + "stdev": 0.4055143799647105, + "sum": 9.866666666666665 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9974.0, + "mean": 322.2212885154062, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6390.722222222223, + "non_zero_median": 7108.033333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 7470.115999999997, + "percentile999": 9686.764400000047, + "range": 9974.0, + "samples": 119, + "stdev": 1563.338918567421, + "sum": 38344.333333333336 + }, + "pg_stat_database_blks_hit": { + "iqr": 18810.033333333333, + "max": 93458.66666666667, + "mean": 15144.335574229692, + "median": 7601.933333333333, + "min": 1207.9333333333334, + "non_zero_mean": 15144.335574229692, + "non_zero_median": 7601.933333333333, + "percentile25": 2174.7333333333336, + "percentile75": 20984.766666666666, + "percentile90": 37523.05333333333, + "percentile99": 75926.3693333333, + "percentile999": 91495.06013333365, + "range": 92250.73333333334, + "samples": 119, + "stdev": 17843.73301688011, + "sum": 1802175.9333333333 + }, + "pg_stat_database_blks_read": { + "iqr": 1.6666666666666667, + "max": 10.533333333333333, + "mean": 1.3490196078431373, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.918787878787879, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 1.6666666666666667, + "percentile90": 4.48, + "percentile99": 9.843999999999994, + "percentile999": 10.47040000000001, + "range": 10.533333333333333, + "samples": 119, + "stdev": 2.3316653583444222, + "sum": 160.5333333333333 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 119, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 1.6666666666666667, + "max": 10.533333333333333, + "mean": 1.3490196078431373, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.918787878787879, + "non_zero_median": 1.7333333333333334, + "percentile25": 0.0, + "percentile75": 1.6666666666666667, + "percentile90": 4.48, + "percentile99": 9.843999999999994, + "percentile999": 10.47040000000001, + "range": 10.533333333333333, + "samples": 119, + "stdev": 2.3316653583444222, + "sum": 160.5333333333333 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4, + "mean": 0.12605042016806722, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8333333333333334, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.6933333333333337, + "percentile99": 1.2546666666666662, + "percentile999": 1.3842666666666692, + "range": 1.4, + "samples": 119, + "stdev": 0.33013036470448476, + "sum": 15.0 + }, + "pg_stat_database_tup_fetched": { + "iqr": 10653.900000000001, + "max": 47197.666666666664, + "mean": 8052.83081232493, + "median": 3891.6666666666665, + "min": 579.2666666666667, + "non_zero_mean": 8052.83081232493, + "non_zero_median": 3891.6666666666665, + "percentile25": 1121.7666666666667, + "percentile75": 11775.666666666668, + "percentile90": 20135.50666666667, + "percentile99": 38033.50133333332, + "percentile999": 46168.6673333335, + "range": 46618.399999999994, + "samples": 119, + "stdev": 9319.981448054206, + "sum": 958286.8666666662 + }, + "pg_stat_database_tup_inserted": { + "iqr": 16.833333333333336, + "max": 82.2, + "mean": 10.738375350140057, + "median": 0.06666666666666667, + "min": 0.0, + "non_zero_mean": 20.61075268817204, + "non_zero_median": 16.333333333333336, + "percentile25": 0.0, + "percentile75": 16.833333333333336, + "percentile90": 34.2, + "percentile99": 68.91066666666666, + "percentile999": 80.65026666666692, + "range": 82.2, + "samples": 119, + "stdev": 17.87405859328396, + "sum": 1277.8666666666668 + }, + "pg_stat_database_tup_returned": { + "iqr": 15729.2, + "max": 76891.33333333333, + "mean": 12822.39943977591, + "median": 6784.0, + "min": 1367.4, + "non_zero_mean": 12822.39943977591, + "non_zero_median": 6784.0, + "percentile25": 1884.0, + "percentile75": 17613.2, + "percentile90": 35583.146666666675, + "percentile99": 58557.85333333329, + "percentile999": 74862.48853333366, + "range": 75523.93333333333, + "samples": 119, + "stdev": 14500.663199638166, + "sum": 1525865.533333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 19.833333333333336, + "max": 58.666666666666664, + "mean": 11.126610644257703, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 12.374454828660436, + "non_zero_median": 3.0, + "percentile25": 0.13333333333333333, + "percentile75": 19.96666666666667, + "percentile90": 37.06666666666666, + "percentile99": 54.09199999999997, + "percentile999": 58.21040000000007, + "range": 58.666666666666664, + "samples": 119, + "stdev": 15.391477561985107, + "sum": 1324.0666666666666 + }, + "pg_stat_database_xact_commit": { + "iqr": 221.0, + "max": 601.6666666666666, + "mean": 106.58487394957983, + "median": 13.466666666666667, + "min": 2.8, + "non_zero_mean": 106.58487394957983, + "non_zero_median": 13.466666666666667, + "percentile25": 7.366666666666667, + "percentile75": 228.36666666666667, + "percentile90": 301.2133333333334, + "percentile99": 414.0799999999999, + "percentile999": 579.6872000000035, + "range": 598.8666666666667, + "samples": 119, + "stdev": 135.53944379359675, + "sum": 12683.6 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 119, + "stdev": 0.0, + "sum": 7.933333333333325 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.002240896358543417, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.23520000000000513, + "range": 0.26666666666666666, + "samples": 119, + "stdev": 0.024445293254085638, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 79.8, + "max": 147.66666666666666, + "mean": 40.635854341736696, + "median": 15.0, + "min": 0.4, + "non_zero_mean": 40.635854341736696, + "non_zero_median": 15.0, + "percentile25": 3.2666666666666666, + "percentile75": 83.06666666666666, + "percentile90": 108.89333333333335, + "percentile99": 141.5266666666666, + "percentile999": 147.07666666666677, + "range": 147.26666666666665, + "samples": 119, + "stdev": 45.143560741850884, + "sum": 4835.666666666669 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,63,60,62,66,65,64,76,71,69,67,75,72,74,68,77,73,78,86,83,79,90,81,82,84,85,92,88,93,94,91,95,98,96,99,100,97,101,102,103,105,106,107,108,110,109,111,104,114,113,112,115,116,120,123,122,124,121,117,119,126,129,128,134,132,136,138,135,144,141,143,142,137,146,149,151,145,150,152,147,131,154,156,157,158,153,161,162,160,163,167,168,159,165,164,166,169,172,171,170,173,179,174,177,176,180,175,183,178,185,188,193,189,186,181,191,184,195,182,194,187,192,190,197,196,199,201,198,200,202,203,204,207,205,206,208,209,210,211,214,212,216,215,219,217,220,218,222,227,213,225,226,229,230,228,221,231,224,233,232,223,234,235,237,236,239,242,244,245,249,248,247,246,250,252,243,251,253,254,240,259,257,241,256,261,238,263,262,264,266,258,265,267,255,260,268,270,269,272,271,273,277,274,275,278,284,291,280,276,279,282,288,281,290,287,283,289,296,292,285,286,297,294,293,298,295,301,300,299,304,303,302,306,305,309,307,315,308,310,311,313,314,319,312,316,321,320,318,322,324,323,325,317,327,326,328,329,330,332,335,337,333,334,336,338,341,346,339,342,344,345,343,349,331,354,348,356,347,353,359,351,352,355,360,357,362,358,363,367,366,340,364,350,368,369,361,365,370,371,372,373,375,382,385,376,378,377,389,374,390,380,384,379,381,392,391,394,387,386,383,388,393,397,400,395,402,403,401,405,398,408,404,407,396,399,406,412,410,411,409,414,413,415,416,417,418,425,419,420,421,423,429,427,426,424,422,433,430,431,434,428,438,432,437,441,439,436,435,440,447,443,444,442,446,451,448,445,449,455,453,454,452,459,460,458,456,462,463,461,466,467,465,468,457,469,450,464,470,471,472,473,474,480,477,475,483,479,482,476,491,478,484,485,481,486,487,490,493,489,488,492,495,499,497,496,498,494,500,503,501,507,502,505,508,504,506,509,515,511,510,512,514,518,517,516,513,519,521,526,522,523,525,532,531,530,520,527,533,536,540,535,528,538,539,537,534,545,541,543,548,542,547,546,544,549,524,551,550,529,552,553,555,558,556,557,554,560,561,563,562,559,565,564,568,567,569,566,571,570,572", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-12T14:13:54.965658Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2022-01-12T14:13:59.417646Z 2022-01-12T14:13:59.261450Z 2022-01-12T14:13:59.224203Z 2022-01-12T14:13:59.225547Z 2022-01-12T14:13:59.073462Z 2022-01-12T14:13:59.071942Z 2022-01-12T14:13:58.984941Z 2022-01-12T14:13:58.850947Z 2022-01-12T14:13:58.822281Z 2022-01-12T14:13:58.249057Z 2022-01-12T14:13:57.990903Z 2022-01-12T14:13:57.752586Z 2022-01-12T14:13:57.648023Z 2022-01-12T14:13:57.555585Z 2022-01-12T14:13:57.507164Z 2022-01-12T14:13:57.460243Z 2022-01-12T14:13:57.449412Z 2022-01-12T14:13:56.965165Z 2022-01-12T14:13:56.853592Z 2022-01-12T14:13:56.547457Z 2022-01-12T14:13:56.388963Z 2022-01-12T14:13:56.342792Z 2022-01-12T14:13:56.315532Z 2022-01-12T14:13:56.288144Z 2022-01-12T14:13:56.132133Z 2022-01-12T14:13:56.070208Z 2022-01-12T14:13:55.975358Z 2022-01-12T14:13:55.796120Z 2022-01-12T14:13:55.600474Z 2022-01-12T14:13:55.511385Z 2022-01-12T14:13:55.385064Z 2022-01-12T14:13:55.383963Z 2022-01-12T14:13:55.321002Z 2022-01-12T14:13:55.189888Z 2022-01-12T14:13:55.072871Z 2022-01-12T14:13:54.965658Z 2022-01-12T14:14:30.494498Z 2022-01-12T14:14:28.935472Z 2022-01-12T14:14:28.842519Z 2022-01-12T14:14:26.705254Z 2022-01-12T14:14:22.601521Z 2022-01-12T14:14:21.013317Z 2022-01-12T14:14:19.227320Z 2022-01-12T14:14:18.803374Z 2022-01-12T14:14:18.789203Z 2022-01-12T14:14:17.149974Z 2022-01-12T14:14:16.309083Z 2022-01-12T14:14:15.942017Z 2022-01-12T14:14:15.075374Z 2022-01-12T14:14:14.874038Z 2022-01-12T14:14:11.589194Z 2022-01-12T14:14:10.912611Z 2022-01-12T14:14:10.279250Z 2022-01-12T14:14:09.669731Z 2022-01-12T14:14:09.279872Z 2022-01-12T14:14:08.026638Z 2022-01-12T14:14:07.686868Z 2022-01-12T14:14:07.547526Z 2022-01-12T14:14:07.469172Z 2022-01-12T14:14:07.251484Z 2022-01-12T14:14:07.080893Z 2022-01-12T14:14:06.848919Z 2022-01-12T14:14:06.586747Z 2022-01-12T14:14:06.485666Z 2022-01-12T14:14:06.457215Z 2022-01-12T14:14:06.296602Z 2022-01-12T14:14:05.910897Z 2022-01-12T14:14:05.586973Z 2022-01-12T14:14:05.548644Z 2022-01-12T14:14:05.489211Z 2022-01-12T14:14:05.410478Z 2022-01-12T14:14:04.988870Z 2022-01-12T14:14:04.771285Z 2022-01-12T14:14:04.340614Z 2022-01-12T14:14:04.242083Z 2022-01-12T14:14:03.798591Z 2022-01-12T14:14:03.722865Z 2022-01-12T14:14:03.470634Z 2022-01-12T14:14:03.173646Z 2022-01-12T14:14:03.125937Z 2022-01-12T14:14:03.124931Z 2022-01-12T14:14:03.068308Z 2022-01-12T14:14:02.755877Z 2022-01-12T14:14:02.609151Z 2022-01-12T14:14:02.606935Z 2022-01-12T14:14:02.118556Z 2022-01-12T14:14:02.081231Z 2022-01-12T14:14:01.946755Z 2022-01-12T14:14:01.790759Z 2022-01-12T14:14:01.621834Z 2022-01-12T14:14:01.393723Z 2022-01-12T14:14:00.926360Z 2022-01-12T14:14:00.724813Z 2022-01-12T14:14:00.597034Z 2022-01-12T14:14:00.481804Z 2022-01-12T14:14:00.394339Z 2022-01-12T14:14:00.341015Z 2022-01-12T14:14:00.291211Z 2022-01-12T14:14:00.102072Z 2022-01-12T14:14:00.025148Z 2022-01-12T14:27:26.853580Z 2022-01-12T14:27:26.685049Z 2022-01-12T14:27:26.488749Z 2022-01-12T14:27:25.679320Z 2022-01-12T14:27:25.656215Z 2022-01-12T14:27:25.526095Z 2022-01-12T14:27:23.822561Z 2022-01-12T14:27:23.436787Z 2022-01-12T14:27:23.340307Z 2022-01-12T14:27:22.429259Z 2022-01-12T14:27:22.293519Z 2022-01-12T14:27:21.801994Z 2022-01-12T14:27:21.350934Z 2022-01-12T14:27:21.034374Z 2022-01-12T14:27:21.002721Z 2022-01-12T14:27:20.837643Z 2022-01-12T14:27:20.196912Z 2022-01-12T14:27:19.984650Z 2022-01-12T14:27:19.564672Z 2022-01-12T14:27:18.409152Z 2022-01-12T14:27:18.366662Z 2022-01-12T14:27:17.277388Z 2022-01-12T14:27:17.138687Z 2022-01-12T14:27:16.907386Z 2022-01-12T14:27:16.740462Z 2022-01-12T14:27:16.661201Z 2022-01-12T14:27:16.470005Z 2022-01-12T14:27:16.340120Z 2022-01-12T14:27:15.365662Z 2022-01-12T14:27:15.364869Z 2022-01-12T14:27:15.299222Z 2022-01-12T14:27:15.252618Z 2022-01-12T14:27:14.922524Z 2022-01-12T14:27:14.488156Z 2022-01-12T14:27:14.005179Z 2022-01-12T14:27:14.032025Z 2022-01-12T14:27:13.901438Z 2022-01-12T14:27:13.703966Z 2022-01-12T14:27:13.531013Z 2022-01-12T14:27:13.227230Z 2022-01-12T14:27:13.068894Z 2022-01-12T14:27:12.452733Z 2022-01-12T14:27:12.366376Z 2022-01-12T14:27:11.518253Z 2022-01-12T14:27:11.305081Z 2022-01-12T14:27:10.762830Z 2022-01-12T14:27:10.297348Z 2022-01-12T14:27:10.253671Z 2022-01-12T14:27:10.190926Z 2022-01-12T14:27:10.043658Z 2022-01-12T14:27:10.029899Z 2022-01-12T14:27:09.389531Z 2022-01-12T14:27:09.289621Z 2022-01-12T14:27:09.098544Z 2022-01-12T14:27:08.982143Z 2022-01-12T14:27:08.649378Z 2022-01-12T14:27:08.199690Z 2022-01-12T14:27:08.185377Z 2022-01-12T14:27:08.001325Z 2022-01-12T14:27:07.838166Z 2022-01-12T14:27:07.730770Z 2022-01-12T14:27:07.374328Z 2022-01-12T14:27:06.537647Z 2022-01-12T14:27:06.479598Z 2022-01-12T14:27:06.475497Z 2022-01-12T14:27:06.264637Z 2022-01-12T14:27:06.239332Z 2022-01-12T14:27:05.574029Z 2022-01-12T14:27:05.514522Z 2022-01-12T14:27:05.334269Z 2022-01-12T14:27:05.153594Z 2022-01-12T14:27:04.596466Z 2022-01-12T14:27:04.591304Z 2022-01-12T14:27:04.511160Z 2022-01-12T14:27:04.380151Z 2022-01-12T14:27:04.141794Z 2022-01-12T14:27:03.989661Z 2022-01-12T14:27:03.748443Z 2022-01-12T14:27:03.259670Z 2022-01-12T14:27:03.176978Z 2022-01-12T14:27:03.092968Z 2022-01-12T14:27:03.066700Z 2022-01-12T14:27:02.969435Z 2022-01-12T14:27:02.755231Z 2022-01-12T14:27:02.644707Z 2022-01-12T14:27:02.567222Z 2022-01-12T14:27:02.198898Z 2022-01-12T14:27:01.970648Z 2022-01-12T14:27:01.915572Z 2022-01-12T14:27:01.893336Z 2022-01-12T14:27:01.831127Z 2022-01-12T14:27:01.737591Z 2022-01-12T14:27:01.642411Z 2022-01-12T14:27:01.387551Z 2022-01-12T14:27:01.376552Z 2022-01-12T14:27:00.897453Z 2022-01-12T14:27:00.673177Z 2022-01-12T14:27:00.554558Z 2022-01-12T14:27:00.249529Z 2022-01-12T14:26:59.980557Z 2022-01-12T14:31:48.429465Z 2022-01-12T14:31:46.546690Z 2022-01-12T14:31:45.402982Z 2022-01-12T14:31:45.044505Z 2022-01-12T14:31:43.687300Z 2022-01-12T14:31:43.679279Z 2022-01-12T14:31:43.629818Z 2022-01-12T14:31:43.200537Z 2022-01-12T14:31:42.658945Z 2022-01-12T14:31:42.476956Z 2022-01-12T14:31:42.276391Z 2022-01-12T14:31:41.675333Z 2022-01-12T14:31:41.306502Z 2022-01-12T14:31:40.962262Z 2022-01-12T14:31:40.821558Z 2022-01-12T14:31:40.708966Z 2022-01-12T14:31:40.367165Z 2022-01-12T14:31:40.276431Z 2022-01-12T14:31:40.217947Z 2022-01-12T14:31:40.118901Z 2022-01-12T14:31:39.987624Z 2022-01-12T14:31:39.679061Z 2022-01-12T14:31:39.634647Z 2022-01-12T14:31:39.518228Z 2022-01-12T14:31:39.266527Z 2022-01-12T14:31:39.200928Z 2022-01-12T14:31:39.092404Z 2022-01-12T14:31:39.065562Z 2022-01-12T14:31:38.968142Z 2022-01-12T14:31:38.863038Z 2022-01-12T14:31:38.596126Z 2022-01-12T14:31:38.240594Z 2022-01-12T14:31:38.231649Z 2022-01-12T14:31:37.774033Z 2022-01-12T14:31:37.555109Z 2022-01-12T14:31:37.385079Z 2022-01-12T14:31:37.355541Z 2022-01-12T14:31:37.263911Z 2022-01-12T14:31:37.206073Z 2022-01-12T14:31:37.201537Z 2022-01-12T14:31:37.148508Z 2022-01-12T14:31:37.138254Z 2022-01-12T14:31:36.915031Z 2022-01-12T14:31:36.909177Z 2022-01-12T14:31:36.818972Z 2022-01-12T14:31:36.247012Z 2022-01-12T14:31:36.135042Z 2022-01-12T14:31:35.750933Z 2022-01-12T14:31:35.649603Z 2022-01-12T14:31:35.618098Z 2022-01-12T14:31:35.280279Z 2022-01-12T14:31:34.991075Z 2022-01-12T14:31:34.891388Z 2022-01-12T14:31:34.764978Z 2022-01-12T14:31:34.761408Z 2022-01-12T14:31:34.704642Z 2022-01-12T14:31:34.603804Z 2022-01-12T14:31:34.419145Z 2022-01-12T14:31:34.400806Z 2022-01-12T14:31:34.303926Z 2022-01-12T14:31:34.294981Z 2022-01-12T14:31:34.178257Z 2022-01-12T14:31:34.116643Z 2022-01-12T14:31:33.791565Z 2022-01-12T14:31:33.666560Z 2022-01-12T14:31:33.349258Z 2022-01-12T14:31:33.044005Z 2022-01-12T14:31:32.850791Z 2022-01-12T14:31:32.837718Z 2022-01-12T14:31:32.798260Z 2022-01-12T14:31:32.673452Z 2022-01-12T14:31:32.491612Z 2022-01-12T14:31:32.305676Z 2022-01-12T14:31:32.273153Z 2022-01-12T14:31:31.980542Z 2022-01-12T14:31:31.840708Z 2022-01-12T14:31:31.835998Z 2022-01-12T14:31:31.670039Z 2022-01-12T14:31:31.626985Z 2022-01-12T14:31:31.561617Z 2022-01-12T14:31:31.416848Z 2022-01-12T14:31:31.297700Z 2022-01-12T14:31:31.148421Z 2022-01-12T14:31:31.030115Z 2022-01-12T14:31:30.953555Z 2022-01-12T14:31:30.833533Z 2022-01-12T14:31:30.806909Z 2022-01-12T14:31:30.655667Z 2022-01-12T14:31:30.625810Z 2022-01-12T14:31:30.619887Z 2022-01-12T14:31:30.535885Z 2022-01-12T14:31:30.465997Z 2022-01-12T14:31:30.419999Z 2022-01-12T14:31:30.275694Z 2022-01-12T14:31:29.957381Z 2022-01-12T14:31:29.895145Z 2022-01-12T14:31:29.778720Z 2022-01-12T14:31:29.545926Z 2022-01-12T14:31:29.486006Z 2022-01-12T14:31:29.305550Z 2022-01-12T14:36:18.380767Z 2022-01-12T14:36:18.045478Z 2022-01-12T14:36:17.964867Z 2022-01-12T14:36:15.649945Z 2022-01-12T14:36:15.209435Z 2022-01-12T14:36:14.939919Z 2022-01-12T14:36:14.882610Z 2022-01-12T14:36:14.839238Z 2022-01-12T14:36:14.822372Z 2022-01-12T14:36:14.341142Z 2022-01-12T14:36:13.815082Z 2022-01-12T14:36:13.693647Z 2022-01-12T14:36:13.624800Z 2022-01-12T14:36:13.437967Z 2022-01-12T14:36:13.305852Z 2022-01-12T14:36:13.287631Z 2022-01-12T14:36:13.220534Z 2022-01-12T14:36:12.901925Z 2022-01-12T14:36:12.808585Z 2022-01-12T14:36:12.736069Z 2022-01-12T14:36:12.694856Z 2022-01-12T14:36:12.686015Z 2022-01-12T14:36:12.646901Z 2022-01-12T14:36:12.593185Z 2022-01-12T14:36:12.462978Z 2022-01-12T14:36:12.273544Z 2022-01-12T14:36:12.235053Z 2022-01-12T14:36:12.060271Z 2022-01-12T14:36:12.022703Z 2022-01-12T14:36:11.836984Z 2022-01-12T14:36:11.588545Z 2022-01-12T14:36:11.587568Z 2022-01-12T14:36:11.387279Z 2022-01-12T14:36:11.058089Z 2022-01-12T14:36:10.858034Z 2022-01-12T14:36:10.781854Z 2022-01-12T14:36:10.159006Z 2022-01-12T14:36:10.056167Z 2022-01-12T14:36:09.914170Z 2022-01-12T14:36:09.905165Z 2022-01-12T14:36:09.801155Z 2022-01-12T14:36:09.783376Z 2022-01-12T14:36:09.771875Z 2022-01-12T14:36:09.683719Z 2022-01-12T14:36:09.703586Z 2022-01-12T14:36:09.672218Z 2022-01-12T14:36:09.491132Z 2022-01-12T14:36:09.478438Z 2022-01-12T14:36:09.343951Z 2022-01-12T14:36:09.213331Z 2022-01-12T14:36:09.155576Z 2022-01-12T14:36:09.079223Z 2022-01-12T14:36:09.059592Z 2022-01-12T14:36:08.922887Z 2022-01-12T14:36:08.693475Z 2022-01-12T14:36:08.439892Z 2022-01-12T14:36:08.292878Z 2022-01-12T14:36:08.156191Z 2022-01-12T14:36:07.834926Z 2022-01-12T14:36:07.791263Z 2022-01-12T14:36:07.716161Z 2022-01-12T14:36:07.531957Z 2022-01-12T14:36:07.420442Z 2022-01-12T14:36:07.380992Z 2022-01-12T14:36:07.039629Z 2022-01-12T14:36:07.039450Z 2022-01-12T14:36:07.039546Z 2022-01-12T14:36:06.898378Z 2022-01-12T14:36:06.881345Z 2022-01-12T14:36:06.878066Z 2022-01-12T14:36:06.837018Z 2022-01-12T14:36:06.827855Z 2022-01-12T14:36:06.781573Z 2022-01-12T14:36:06.809963Z 2022-01-12T14:36:06.776533Z 2022-01-12T14:36:06.702287Z 2022-01-12T14:36:06.571005Z 2022-01-12T14:36:06.496286Z 2022-01-12T14:36:06.254104Z 2022-01-12T14:36:06.133770Z 2022-01-12T14:36:05.941879Z 2022-01-12T14:36:05.917428Z 2022-01-12T14:36:05.799362Z 2022-01-12T14:36:05.722668Z 2022-01-12T14:36:05.700100Z 2022-01-12T14:36:05.651621Z 2022-01-12T14:36:05.641871Z 2022-01-12T14:36:05.439912Z 2022-01-12T14:36:05.410706Z 2022-01-12T14:36:05.376966Z 2022-01-12T14:36:05.098062Z 2022-01-12T14:36:04.959129Z 2022-01-12T14:36:04.890314Z 2022-01-12T14:36:04.841948Z 2022-01-12T14:36:04.695425Z 2022-01-12T14:36:04.535307Z 2022-01-12T14:36:04.526177Z 2022-01-12T14:36:04.400537Z 2022-01-12T14:36:04.386287Z 2022-01-12T14:36:04.383779Z 2022-01-12T14:41:16.807684Z 2022-01-12T14:41:16.327913Z 2022-01-12T14:41:16.226143Z 2022-01-12T14:41:15.536790Z 2022-01-12T14:41:15.355426Z 2022-01-12T14:41:15.269615Z 2022-01-12T14:41:14.891002Z 2022-01-12T14:41:14.877994Z 2022-01-12T14:41:14.500615Z 2022-01-12T14:41:14.335684Z 2022-01-12T14:41:13.948334Z 2022-01-12T14:41:13.828942Z 2022-01-12T14:41:13.648295Z 2022-01-12T14:41:13.302677Z 2022-01-12T14:41:12.824645Z 2022-01-12T14:41:12.279282Z 2022-01-12T14:41:12.101397Z 2022-01-12T14:41:11.792178Z 2022-01-12T14:41:11.754411Z 2022-01-12T14:41:11.426825Z 2022-01-12T14:41:11.375669Z 2022-01-12T14:41:11.217393Z 2022-01-12T14:41:11.052168Z 2022-01-12T14:41:10.752105Z 2022-01-12T14:41:10.248027Z 2022-01-12T14:41:09.923568Z 2022-01-12T14:41:09.490167Z 2022-01-12T14:41:09.341591Z 2022-01-12T14:41:09.305124Z 2022-01-12T14:41:09.290287Z 2022-01-12T14:41:09.154597Z 2022-01-12T14:41:08.718429Z 2022-01-12T14:41:08.360432Z 2022-01-12T14:41:07.954249Z 2022-01-12T14:41:07.832111Z 2022-01-12T14:41:07.844875Z 2022-01-12T14:41:07.683334Z 2022-01-12T14:41:07.524547Z 2022-01-12T14:41:07.434705Z 2022-01-12T14:41:07.003148Z 2022-01-12T14:41:06.952862Z 2022-01-12T14:41:06.827339Z 2022-01-12T14:41:06.694860Z 2022-01-12T14:41:06.502008Z 2022-01-12T14:41:06.459723Z 2022-01-12T14:41:06.391248Z 2022-01-12T14:41:06.256052Z 2022-01-12T14:41:06.203927Z 2022-01-12T14:41:05.870811Z 2022-01-12T14:41:05.388323Z 2022-01-12T14:41:05.297954Z 2022-01-12T14:41:05.238983Z 2022-01-12T14:41:05.208579Z 2022-01-12T14:41:04.826265Z 2022-01-12T14:41:04.737879Z 2022-01-12T14:41:04.462315Z 2022-01-12T14:41:04.384075Z 2022-01-12T14:41:04.254171Z 2022-01-12T14:41:04.124498Z 2022-01-12T14:41:03.970798Z 2022-01-12T14:41:03.784255Z 2022-01-12T14:41:03.503399Z 2022-01-12T14:41:03.465720Z 2022-01-12T14:41:03.275534Z 2022-01-12T14:41:03.175862Z 2022-01-12T14:41:03.158151Z 2022-01-12T14:41:03.155943Z 2022-01-12T14:41:03.117158Z 2022-01-12T14:41:03.043374Z 2022-01-12T14:41:03.003328Z 2022-01-12T14:41:02.951471Z 2022-01-12T14:41:02.738315Z 2022-01-12T14:41:02.707603Z 2022-01-12T14:41:02.664532Z 2022-01-12T14:41:02.436977Z 2022-01-12T14:41:02.423481Z 2022-01-12T14:41:02.396099Z 2022-01-12T14:41:02.181237Z 2022-01-12T14:41:01.922497Z 2022-01-12T14:41:01.708879Z 2022-01-12T14:41:01.674652Z 2022-01-12T14:41:01.521871Z 2022-01-12T14:41:01.371421Z 2022-01-12T14:41:01.275508Z 2022-01-12T14:41:01.253473Z 2022-01-12T14:41:01.141472Z 2022-01-12T14:41:01.041476Z 2022-01-12T14:41:01.020944Z 2022-01-12T14:41:00.936107Z 2022-01-12T14:41:00.927049Z 2022-01-12T14:41:00.892321Z 2022-01-12T14:41:00.852326Z 2022-01-12T14:41:00.776647Z 2022-01-12T14:41:00.708668Z 2022-01-12T14:41:00.575229Z 2022-01-12T14:41:00.363140Z 2022-01-12T14:41:00.253658Z 2022-01-12T14:41:00.096274Z 2022-01-12T14:40:59.777974Z 2022-01-12T14:40:59.279355Z", + "created_diff": 22.610219, + "ended": "2022-01-12T14:43:31.599156Z", + "error_job_ids": " []", + "event_first": "2022-01-12T14:14:41.503271Z 2022-01-12T14:15:26.044470Z 2022-01-12T14:15:22.246710Z 2022-01-12T14:14:35.912759Z 2022-01-12T14:14:32.637656Z 2022-01-12T14:15:12.996148Z 2022-01-12T14:14:34.232881Z 2022-01-12T14:15:28.550930Z 2022-01-12T14:14:32.087333Z 2022-01-12T14:15:14.968505Z 2022-01-12T14:15:12.696171Z 2022-01-12T14:14:55.783885Z 2022-01-12T14:16:56.667514Z 2022-01-12T14:14:55.380377Z 2022-01-12T14:15:56.389338Z 2022-01-12T14:14:59.575358Z 2022-01-12T14:17:05.705179Z 2022-01-12T14:14:56.391384Z 2022-01-12T14:15:46.806131Z 2022-01-12T14:14:50.076209Z 2022-01-12T14:16:26.923561Z 2022-01-12T14:14:44.154310Z 2022-01-12T14:17:15.667202Z 2022-01-12T14:14:46.226606Z 2022-01-12T14:16:47.554556Z 2022-01-12T14:14:36.755015Z 2022-01-12T14:16:16.494389Z 2022-01-12T14:14:17.521134Z 2022-01-12T14:15:38.006334Z 2022-01-12T14:14:11.805112Z 2022-01-12T14:16:06.915256Z 2022-01-12T14:14:19.845440Z 2022-01-12T14:16:37.599118Z 2022-01-12T14:14:06.198492Z 2022-01-12T14:14:29.797500Z 2022-01-12T14:14:06.531356Z 2022-01-12T14:15:50.582287Z 2022-01-12T14:15:29.050625Z 2022-01-12T14:15:49.312678Z 2022-01-12T14:15:32.146137Z 2022-01-12T14:15:32.754580Z 2022-01-12T14:15:29.459366Z 2022-01-12T14:15:30.985289Z 2022-01-12T14:15:28.443368Z 2022-01-12T14:15:32.671627Z 2022-01-12T14:15:29.744308Z 2022-01-12T14:15:42.538699Z 2022-01-12T14:15:23.257267Z 2022-01-12T14:15:47.738317Z 2022-01-12T14:15:33.002790Z 2022-01-12T14:15:17.631362Z 2022-01-12T14:15:42.330194Z 2022-01-12T14:15:26.161283Z 2022-01-12T14:15:45.192681Z 2022-01-12T14:15:23.555412Z 2022-01-12T14:15:48.255638Z 2022-01-12T14:15:22.853617Z 2022-01-12T14:15:41.828492Z 2022-01-12T14:15:17.881961Z 2022-01-12T14:15:40.609685Z 2022-01-12T14:15:21.874383Z 2022-01-12T14:15:48.256970Z 2022-01-12T14:15:15.916217Z 2022-01-12T14:15:43.678838Z 2022-01-12T14:15:19.904150Z 2022-01-12T14:15:46.731382Z 2022-01-12T14:15:11.553118Z 2022-01-12T14:15:40.616562Z 2022-01-12T14:15:14.635206Z 2022-01-12T14:15:42.301380Z 2022-01-12T14:15:12.045497Z 2022-01-12T14:15:40.091243Z 2022-01-12T14:15:46.230469Z 2022-01-12T14:15:39.837479Z 2022-01-12T14:14:45.377275Z 2022-01-12T14:15:39.054211Z 2022-01-12T14:14:52.243364Z 2022-01-12T14:15:35.807553Z 2022-01-12T14:14:46.393967Z 2022-01-12T14:15:38.695819Z 2022-01-12T14:14:54.491421Z 2022-01-12T14:15:34.680131Z 2022-01-12T14:14:51.261334Z 2022-01-12T14:15:36.593485Z 2022-01-12T14:14:44.964342Z 2022-01-12T14:15:33.477415Z 2022-01-12T14:14:47.982295Z 2022-01-12T14:15:23.804022Z 2022-01-12T14:14:40.807304Z 2022-01-12T14:15:26.782224Z 2022-01-12T14:14:42.816611Z 2022-01-12T14:15:30.757193Z 2022-01-12T14:14:41.503271Z 2022-01-12T14:15:33.817602Z 2022-01-12T14:14:40.312251Z 2022-01-12T14:15:17.635401Z 2022-01-12T14:14:39.227589Z 2022-01-12T14:15:30.512416Z 2022-01-12T14:14:37.207927Z 2022-01-12T14:15:20.357394Z 2022-01-12T14:29:04.978360Z 2022-01-12T14:28:59.069267Z 2022-01-12T14:29:02.285572Z 2022-01-12T14:28:58.358967Z 2022-01-12T14:29:03.234417Z 2022-01-12T14:28:53.855230Z 2022-01-12T14:29:06.974931Z 2022-01-12T14:28:56.527996Z 2022-01-12T14:29:06.458640Z 2022-01-12T14:28:55.699540Z 2022-01-12T14:29:04.607252Z 2022-01-12T14:28:56.818291Z 2022-01-12T14:29:04.607252Z 2022-01-12T14:28:55.973084Z 2022-01-12T14:29:02.549443Z 2022-01-12T14:28:56.351548Z 2022-01-12T14:29:05.226254Z 2022-01-12T14:28:48.704329Z 2022-01-12T14:28:57.608742Z 2022-01-12T14:28:46.049161Z 2022-01-12T14:29:02.848386Z 2022-01-12T14:28:51.731982Z 2022-01-12T14:28:56.351548Z 2022-01-12T14:28:51.280418Z 2022-01-12T14:28:53.067427Z 2022-01-12T14:28:54.134318Z 2022-01-12T14:28:44.868535Z 2022-01-12T14:28:55.513033Z 2022-01-12T14:28:46.049161Z 2022-01-12T14:28:49.488531Z 2022-01-12T14:28:41.477294Z 2022-01-12T14:28:59.854035Z 2022-01-12T14:28:56.502464Z 2022-01-12T14:28:53.855230Z 2022-01-12T14:29:02.122733Z 2022-01-12T14:28:52.282349Z 2022-01-12T14:28:47.857366Z 2022-01-12T14:28:54.894067Z 2022-01-12T14:28:42.196511Z 2022-01-12T14:28:50.823381Z 2022-01-12T14:28:46.049161Z 2022-01-12T14:28:46.474239Z 2022-01-12T14:28:46.049161Z 2022-01-12T14:28:42.816380Z 2022-01-12T14:28:49.144315Z 2022-01-12T14:28:49.226550Z 2022-01-12T14:28:30.628848Z 2022-01-12T14:28:44.632250Z 2022-01-12T14:28:44.507222Z 2022-01-12T14:28:41.067309Z 2022-01-12T14:28:39.270425Z 2022-01-12T14:28:38.029737Z 2022-01-12T14:28:45.230281Z 2022-01-12T14:28:42.816380Z 2022-01-12T14:28:34.184340Z 2022-01-12T14:28:44.149297Z 2022-01-12T14:28:38.452284Z 2022-01-12T14:28:37.845849Z 2022-01-12T14:28:40.646191Z 2022-01-12T14:28:31.842530Z 2022-01-12T14:28:24.312919Z 2022-01-12T14:28:40.232949Z 2022-01-12T14:28:32.676131Z 2022-01-12T14:28:34.064644Z 2022-01-12T14:28:30.373397Z 2022-01-12T14:28:26.831116Z 2022-01-12T14:28:32.913451Z 2022-01-12T14:28:39.518372Z 2022-01-12T14:28:17.570486Z 2022-01-12T14:28:17.484434Z 2022-01-12T14:28:18.685502Z 2022-01-12T14:28:14.533884Z 2022-01-12T14:28:24.117236Z 2022-01-12T14:28:17.927181Z 2022-01-12T14:28:27.903530Z 2022-01-12T14:28:05.111542Z 2022-01-12T14:27:49.374968Z 2022-01-12T14:27:55.543431Z 2022-01-12T14:27:44.991812Z 2022-01-12T14:27:55.037180Z 2022-01-12T14:27:43.791354Z 2022-01-12T14:27:48.385028Z 2022-01-12T14:27:40.073684Z 2022-01-12T14:27:47.554156Z 2022-01-12T14:27:32.784699Z 2022-01-12T14:27:39.071926Z 2022-01-12T14:27:40.674510Z 2022-01-12T14:27:33.666045Z 2022-01-12T14:27:32.237282Z 2022-01-12T14:27:29.617664Z 2022-01-12T14:27:21.632954Z 2022-01-12T14:27:20.397454Z 2022-01-12T14:27:21.441428Z 2022-01-12T14:27:29.140394Z 2022-01-12T14:27:21.686684Z 2022-01-12T14:27:18.350869Z 2022-01-12T14:27:13.451127Z 2022-01-12T14:27:19.636235Z 2022-01-12T14:27:13.222605Z 2022-01-12T14:27:12.892074Z 2022-01-12T14:33:36.174329Z 2022-01-12T14:33:23.685349Z 2022-01-12T14:33:35.140438Z 2022-01-12T14:33:23.909517Z 2022-01-12T14:33:25.416674Z 2022-01-12T14:33:19.095278Z 2022-01-12T14:33:23.315543Z 2022-01-12T14:33:20.362142Z 2022-01-12T14:33:28.988225Z 2022-01-12T14:33:20.360633Z 2022-01-12T14:33:32.659645Z 2022-01-12T14:33:21.823108Z 2022-01-12T14:33:34.682658Z 2022-01-12T14:33:19.339429Z 2022-01-12T14:33:29.275038Z 2022-01-12T14:33:17.375406Z 2022-01-12T14:33:28.147079Z 2022-01-12T14:33:20.096539Z 2022-01-12T14:33:30.929945Z 2022-01-12T14:33:23.329569Z 2022-01-12T14:33:30.357905Z 2022-01-12T14:33:21.819407Z 2022-01-12T14:33:27.949484Z 2022-01-12T14:33:20.360633Z 2022-01-12T14:33:30.618212Z 2022-01-12T14:33:18.840998Z 2022-01-12T14:33:28.741884Z 2022-01-12T14:33:14.696323Z 2022-01-12T14:33:30.344268Z 2022-01-12T14:33:12.096807Z 2022-01-12T14:33:28.400263Z 2022-01-12T14:33:11.771598Z 2022-01-12T14:33:21.487491Z 2022-01-12T14:33:10.963489Z 2022-01-12T14:33:21.798826Z 2022-01-12T14:33:10.198208Z 2022-01-12T14:33:23.906534Z 2022-01-12T14:33:14.283073Z 2022-01-12T14:33:25.456007Z 2022-01-12T14:33:09.379590Z 2022-01-12T14:33:22.498224Z 2022-01-12T14:33:14.580164Z 2022-01-12T14:33:19.590282Z 2022-01-12T14:33:10.447931Z 2022-01-12T14:33:18.313219Z 2022-01-12T14:33:07.508154Z 2022-01-12T14:33:19.792877Z 2022-01-12T14:33:06.904006Z 2022-01-12T14:33:18.777693Z 2022-01-12T14:33:09.127596Z 2022-01-12T14:33:13.482034Z 2022-01-12T14:33:01.527603Z 2022-01-12T14:33:16.173807Z 2022-01-12T14:33:09.379590Z 2022-01-12T14:33:05.881793Z 2022-01-12T14:33:05.077690Z 2022-01-12T14:33:15.164605Z 2022-01-12T14:33:00.719576Z 2022-01-12T14:33:15.418261Z 2022-01-12T14:33:06.331034Z 2022-01-12T14:33:02.118710Z 2022-01-12T14:33:00.424666Z 2022-01-12T14:33:14.405740Z 2022-01-12T14:32:54.802448Z 2022-01-12T14:33:01.998297Z 2022-01-12T14:32:51.571026Z 2022-01-12T14:32:57.474142Z 2022-01-12T14:32:53.798367Z 2022-01-12T14:33:06.491097Z 2022-01-12T14:32:56.401750Z 2022-01-12T14:32:59.343390Z 2022-01-12T14:32:34.525392Z 2022-01-12T14:32:47.721127Z 2022-01-12T14:32:47.655157Z 2022-01-12T14:32:08.317031Z 2022-01-12T14:32:12.726622Z 2022-01-12T14:32:15.033096Z 2022-01-12T14:32:40.559869Z 2022-01-12T14:32:09.734478Z 2022-01-12T14:32:27.334703Z 2022-01-12T14:32:07.871944Z 2022-01-12T14:32:17.364008Z 2022-01-12T14:32:06.766507Z 2022-01-12T14:32:25.399734Z 2022-01-12T14:32:03.068529Z 2022-01-12T14:32:15.110609Z 2022-01-12T14:32:04.344924Z 2022-01-12T14:32:18.656610Z 2022-01-12T14:31:58.121250Z 2022-01-12T14:32:18.197385Z 2022-01-12T14:31:55.669852Z 2022-01-12T14:32:04.691004Z 2022-01-12T14:31:53.617369Z 2022-01-12T14:32:01.738501Z 2022-01-12T14:31:46.275978Z 2022-01-12T14:31:48.594063Z 2022-01-12T14:31:43.364529Z 2022-01-12T14:31:43.696525Z 2022-01-12T14:31:41.581625Z 2022-01-12T14:31:43.471909Z 2022-01-12T14:37:58.461313Z 2022-01-12T14:38:05.007350Z 2022-01-12T14:37:56.423568Z 2022-01-12T14:38:08.575227Z 2022-01-12T14:37:55.281668Z 2022-01-12T14:38:06.912053Z 2022-01-12T14:37:55.477326Z 2022-01-12T14:38:06.296236Z 2022-01-12T14:37:56.782135Z 2022-01-12T14:38:07.055367Z 2022-01-12T14:37:53.468654Z 2022-01-12T14:38:04.858173Z 2022-01-12T14:37:53.601558Z 2022-01-12T14:38:02.965556Z 2022-01-12T14:37:52.952177Z 2022-01-12T14:38:09.087689Z 2022-01-12T14:37:51.824342Z 2022-01-12T14:38:00.739745Z 2022-01-12T14:37:50.425191Z 2022-01-12T14:37:55.946052Z 2022-01-12T14:37:50.318492Z 2022-01-12T14:37:59.639988Z 2022-01-12T14:37:43.178567Z 2022-01-12T14:37:58.552443Z 2022-01-12T14:37:51.824342Z 2022-01-12T14:38:02.974680Z 2022-01-12T14:37:43.290222Z 2022-01-12T14:38:00.395367Z 2022-01-12T14:37:47.500185Z 2022-01-12T14:38:03.024117Z 2022-01-12T14:37:44.187371Z 2022-01-12T14:38:00.144196Z 2022-01-12T14:37:41.729700Z 2022-01-12T14:37:53.153999Z 2022-01-12T14:37:50.947274Z 2022-01-12T14:38:01.000447Z 2022-01-12T14:37:43.178567Z 2022-01-12T14:37:56.371209Z 2022-01-12T14:37:50.425191Z 2022-01-12T14:38:00.395367Z 2022-01-12T14:37:44.700177Z 2022-01-12T14:37:59.887916Z 2022-01-12T14:37:42.951205Z 2022-01-12T14:37:45.477634Z 2022-01-12T14:37:53.394978Z 2022-01-12T14:37:56.927499Z 2022-01-12T14:37:39.860497Z 2022-01-12T14:37:45.373308Z 2022-01-12T14:37:43.687255Z 2022-01-12T14:37:49.786253Z 2022-01-12T14:37:32.471239Z 2022-01-12T14:37:44.011289Z 2022-01-12T14:37:36.761935Z 2022-01-12T14:37:45.126580Z 2022-01-12T14:37:39.671478Z 2022-01-12T14:37:41.728463Z 2022-01-12T14:37:31.174916Z 2022-01-12T14:37:41.095496Z 2022-01-12T14:37:31.154953Z 2022-01-12T14:37:43.667470Z 2022-01-12T14:37:24.141028Z 2022-01-12T14:37:35.999629Z 2022-01-12T14:37:23.892594Z 2022-01-12T14:37:34.528227Z 2022-01-12T14:37:24.834651Z 2022-01-12T14:37:18.578282Z 2022-01-12T14:37:42.743860Z 2022-01-12T14:37:34.957657Z 2022-01-12T14:37:15.069383Z 2022-01-12T14:37:38.122348Z 2022-01-12T14:37:10.551898Z 2022-01-12T14:37:28.568074Z 2022-01-12T14:37:41.607293Z 2022-01-12T14:37:11.799173Z 2022-01-12T14:37:13.368190Z 2022-01-12T14:37:25.311272Z 2022-01-12T14:37:00.858694Z 2022-01-12T14:37:22.262817Z 2022-01-12T14:36:57.480006Z 2022-01-12T14:36:43.270700Z 2022-01-12T14:36:32.445642Z 2022-01-12T14:36:47.869201Z 2022-01-12T14:36:32.203653Z 2022-01-12T14:36:39.706214Z 2022-01-12T14:36:33.391018Z 2022-01-12T14:36:38.675822Z 2022-01-12T14:36:30.342547Z 2022-01-12T14:36:43.727660Z 2022-01-12T14:36:29.608421Z 2022-01-12T14:36:31.420281Z 2022-01-12T14:36:28.787081Z 2022-01-12T14:36:30.982004Z 2022-01-12T14:36:27.523415Z 2022-01-12T14:36:37.653231Z 2022-01-12T14:36:27.784539Z 2022-01-12T14:36:25.130107Z 2022-01-12T14:36:18.885773Z 2022-01-12T14:36:20.020047Z 2022-01-12T14:36:16.551419Z 2022-01-12T14:36:20.746095Z 2022-01-12T14:43:25.914050Z 2022-01-12T14:43:12.415172Z 2022-01-12T14:43:25.672425Z 2022-01-12T14:43:08.620040Z 2022-01-12T14:43:27.150793Z 2022-01-12T14:43:16.364152Z 2022-01-12T14:43:26.388472Z 2022-01-12T14:43:13.227183Z 2022-01-12T14:43:27.966060Z 2022-01-12T14:43:18.145847Z 2022-01-12T14:43:27.146392Z 2022-01-12T14:43:15.860861Z 2022-01-12T14:43:25.689660Z 2022-01-12T14:43:15.023418Z 2022-01-12T14:43:25.901429Z 2022-01-12T14:43:17.067322Z 2022-01-12T14:43:13.593353Z 2022-01-12T14:43:09.376550Z 2022-01-12T14:43:21.832793Z 2022-01-12T14:43:16.919697Z 2022-01-12T14:43:28.875918Z 2022-01-12T14:43:09.042792Z 2022-01-12T14:43:23.871338Z 2022-01-12T14:43:10.125168Z 2022-01-12T14:43:22.847939Z 2022-01-12T14:43:06.056036Z 2022-01-12T14:43:18.514615Z 2022-01-12T14:43:07.070788Z 2022-01-12T14:43:18.455302Z 2022-01-12T14:43:09.633519Z 2022-01-12T14:43:11.169531Z 2022-01-12T14:43:06.351681Z 2022-01-12T14:43:19.474336Z 2022-01-12T14:43:10.912693Z 2022-01-12T14:43:04.064030Z 2022-01-12T14:43:15.658661Z 2022-01-12T14:43:19.882543Z 2022-01-12T14:43:05.812479Z 2022-01-12T14:43:26.151582Z 2022-01-12T14:43:05.991860Z 2022-01-12T14:43:16.252186Z 2022-01-12T14:43:11.024072Z 2022-01-12T14:42:56.033713Z 2022-01-12T14:43:09.124327Z 2022-01-12T14:43:09.246868Z 2022-01-12T14:42:54.541060Z 2022-01-12T14:43:06.416609Z 2022-01-12T14:42:54.029761Z 2022-01-12T14:43:06.606357Z 2022-01-12T14:42:57.501832Z 2022-01-12T14:42:48.291419Z 2022-01-12T14:42:59.907524Z 2022-01-12T14:42:55.561550Z 2022-01-12T14:42:57.491417Z 2022-01-12T14:42:46.984345Z 2022-01-12T14:43:02.433297Z 2022-01-12T14:42:53.249370Z 2022-01-12T14:43:02.600334Z 2022-01-12T14:42:44.811906Z 2022-01-12T14:42:55.351656Z 2022-01-12T14:42:43.903927Z 2022-01-12T14:42:56.688459Z 2022-01-12T14:42:47.571097Z 2022-01-12T14:42:43.337521Z 2022-01-12T14:42:36.168370Z 2022-01-12T14:42:46.391372Z 2022-01-12T14:42:42.254392Z 2022-01-12T14:42:45.221229Z 2022-01-12T14:42:13.673717Z 2022-01-12T14:42:40.450778Z 2022-01-12T14:42:35.402187Z 2022-01-12T14:42:52.795345Z 2022-01-12T14:42:31.287919Z 2022-01-12T14:42:38.881598Z 2022-01-12T14:42:21.244334Z 2022-01-12T14:42:49.043810Z 2022-01-12T14:42:18.063592Z 2022-01-12T14:42:30.942886Z 2022-01-12T14:42:29.835594Z 2022-01-12T14:42:27.778619Z 2022-01-12T14:42:33.752212Z 2022-01-12T14:42:35.214666Z 2022-01-12T14:42:20.942671Z 2022-01-12T14:42:22.288137Z 2022-01-12T14:42:20.107292Z 2022-01-12T14:42:11.628778Z 2022-01-12T14:42:09.064806Z 2022-01-12T14:42:25.111840Z 2022-01-12T14:42:15.811386Z 2022-01-12T14:42:19.315929Z 2022-01-12T14:41:24.698398Z 2022-01-12T14:41:38.930046Z 2022-01-12T14:41:28.200851Z 2022-01-12T14:41:23.975350Z 2022-01-12T14:41:23.950901Z 2022-01-12T14:41:34.287868Z 2022-01-12T14:41:25.705068Z 2022-01-12T14:41:26.509512Z 2022-01-12T14:41:12.103183Z 2022-01-12T14:41:09.955923Z", + "event_last": "2022-01-12T14:14:44.659711Z 2022-01-12T14:15:30.563848Z 2022-01-12T14:15:26.124075Z 2022-01-12T14:14:37.728992Z 2022-01-12T14:14:34.893886Z 2022-01-12T14:15:17.073148Z 2022-01-12T14:14:36.441835Z 2022-01-12T14:15:33.442715Z 2022-01-12T14:14:34.078959Z 2022-01-12T14:15:18.793076Z 2022-01-12T14:15:15.169093Z 2022-01-12T14:15:02.200079Z 2022-01-12T14:16:57.127763Z 2022-01-12T14:15:02.132020Z 2022-01-12T14:15:56.854627Z 2022-01-12T14:15:04.852528Z 2022-01-12T14:17:06.125109Z 2022-01-12T14:15:01.338098Z 2022-01-12T14:15:47.338954Z 2022-01-12T14:14:55.901951Z 2022-01-12T14:16:28.962845Z 2022-01-12T14:14:49.827521Z 2022-01-12T14:17:16.069116Z 2022-01-12T14:14:53.868316Z 2022-01-12T14:16:47.994697Z 2022-01-12T14:14:42.143139Z 2022-01-12T14:16:16.682590Z 2022-01-12T14:14:20.490156Z 2022-01-12T14:15:38.469958Z 2022-01-12T14:14:14.890323Z 2022-01-12T14:16:07.848718Z 2022-01-12T14:14:22.489344Z 2022-01-12T14:16:39.199592Z 2022-01-12T14:14:08.155484Z 2022-01-12T14:14:32.292004Z 2022-01-12T14:14:08.762182Z 2022-01-12T14:15:52.828679Z 2022-01-12T14:15:31.257007Z 2022-01-12T14:15:51.338088Z 2022-01-12T14:15:33.833524Z 2022-01-12T14:15:34.607199Z 2022-01-12T14:15:31.257007Z 2022-01-12T14:15:32.753101Z 2022-01-12T14:15:30.311976Z 2022-01-12T14:15:34.444108Z 2022-01-12T14:15:31.592412Z 2022-01-12T14:15:45.165020Z 2022-01-12T14:15:26.805017Z 2022-01-12T14:15:50.345396Z 2022-01-12T14:15:34.770600Z 2022-01-12T14:15:20.653034Z 2022-01-12T14:15:46.118950Z 2022-01-12T14:15:28.685542Z 2022-01-12T14:15:48.529079Z 2022-01-12T14:15:27.008330Z 2022-01-12T14:15:50.657964Z 2022-01-12T14:15:26.030495Z 2022-01-12T14:15:45.545167Z 2022-01-12T14:15:21.344114Z 2022-01-12T14:15:43.443837Z 2022-01-12T14:15:23.973976Z 2022-01-12T14:15:50.481202Z 2022-01-12T14:15:19.053854Z 2022-01-12T14:15:47.325030Z 2022-01-12T14:15:22.864647Z 2022-01-12T14:15:49.082983Z 2022-01-12T14:15:15.435540Z 2022-01-12T14:15:44.004158Z 2022-01-12T14:15:18.015958Z 2022-01-12T14:15:45.660877Z 2022-01-12T14:15:14.717093Z 2022-01-12T14:15:44.041456Z 2022-01-12T14:15:48.503005Z 2022-01-12T14:15:43.451609Z 2022-01-12T14:14:49.249361Z 2022-01-12T14:15:43.759571Z 2022-01-12T14:14:54.045234Z 2022-01-12T14:15:39.650032Z 2022-01-12T14:14:49.703125Z 2022-01-12T14:15:42.674253Z 2022-01-12T14:14:56.372615Z 2022-01-12T14:15:38.297091Z 2022-01-12T14:14:53.738806Z 2022-01-12T14:15:40.955428Z 2022-01-12T14:14:48.442319Z 2022-01-12T14:15:36.439341Z 2022-01-12T14:14:51.936979Z 2022-01-12T14:15:29.526959Z 2022-01-12T14:14:43.952819Z 2022-01-12T14:15:30.690078Z 2022-01-12T14:14:46.341091Z 2022-01-12T14:15:35.547796Z 2022-01-12T14:14:44.498168Z 2022-01-12T14:15:37.816046Z 2022-01-12T14:14:43.342384Z 2022-01-12T14:15:21.709996Z 2022-01-12T14:14:42.034684Z 2022-01-12T14:15:35.893017Z 2022-01-12T14:14:39.038957Z 2022-01-12T14:15:24.787057Z 2022-01-12T14:29:07.130366Z 2022-01-12T14:29:01.681846Z 2022-01-12T14:29:05.987272Z 2022-01-12T14:29:00.091832Z 2022-01-12T14:29:06.215083Z 2022-01-12T14:28:56.635198Z 2022-01-12T14:29:08.393598Z 2022-01-12T14:28:58.499205Z 2022-01-12T14:29:07.964715Z 2022-01-12T14:28:58.232890Z 2022-01-12T14:29:07.036450Z 2022-01-12T14:28:59.069267Z 2022-01-12T14:29:07.568210Z 2022-01-12T14:28:58.709446Z 2022-01-12T14:29:05.254066Z 2022-01-12T14:28:58.119194Z 2022-01-12T14:29:07.778046Z 2022-01-12T14:28:53.139097Z 2022-01-12T14:29:02.052123Z 2022-01-12T14:28:50.139815Z 2022-01-12T14:29:06.044940Z 2022-01-12T14:28:54.696289Z 2022-01-12T14:28:59.012405Z 2022-01-12T14:28:57.407113Z 2022-01-12T14:28:55.854659Z 2022-01-12T14:28:57.137018Z 2022-01-12T14:28:48.498985Z 2022-01-12T14:28:59.970574Z 2022-01-12T14:28:49.530768Z 2022-01-12T14:28:55.675429Z 2022-01-12T14:28:45.475955Z 2022-01-12T14:29:02.333476Z 2022-01-12T14:29:00.442087Z 2022-01-12T14:28:56.635198Z 2022-01-12T14:29:05.340111Z 2022-01-12T14:28:55.919544Z 2022-01-12T14:28:51.523143Z 2022-01-12T14:28:59.059428Z 2022-01-12T14:28:45.905006Z 2022-01-12T14:28:56.170929Z 2022-01-12T14:28:49.060016Z 2022-01-12T14:28:51.758151Z 2022-01-12T14:28:50.631388Z 2022-01-12T14:28:47.697084Z 2022-01-12T14:28:53.114067Z 2022-01-12T14:28:53.950192Z 2022-01-12T14:28:36.296010Z 2022-01-12T14:28:49.741190Z 2022-01-12T14:28:48.379658Z 2022-01-12T14:28:46.423072Z 2022-01-12T14:28:42.603227Z 2022-01-12T14:28:43.959319Z 2022-01-12T14:28:50.202970Z 2022-01-12T14:28:47.801284Z 2022-01-12T14:28:38.764996Z 2022-01-12T14:28:50.366041Z 2022-01-12T14:28:40.985956Z 2022-01-12T14:28:44.272037Z 2022-01-12T14:28:44.829994Z 2022-01-12T14:28:35.719043Z 2022-01-12T14:28:28.693801Z 2022-01-12T14:28:45.377730Z 2022-01-12T14:28:36.970003Z 2022-01-12T14:28:38.759931Z 2022-01-12T14:28:35.803505Z 2022-01-12T14:28:31.433151Z 2022-01-12T14:28:37.687445Z 2022-01-12T14:28:45.758063Z 2022-01-12T14:28:22.009075Z 2022-01-12T14:28:22.701053Z 2022-01-12T14:28:22.236305Z 2022-01-12T14:28:20.156506Z 2022-01-12T14:28:27.864516Z 2022-01-12T14:28:21.778921Z 2022-01-12T14:28:33.240060Z 2022-01-12T14:28:11.612070Z 2022-01-12T14:27:52.987508Z 2022-01-12T14:27:58.531959Z 2022-01-12T14:27:49.848012Z 2022-01-12T14:27:58.841248Z 2022-01-12T14:27:48.620965Z 2022-01-12T14:27:53.572012Z 2022-01-12T14:27:44.195920Z 2022-01-12T14:27:52.117189Z 2022-01-12T14:27:35.057666Z 2022-01-12T14:27:42.332833Z 2022-01-12T14:27:45.562198Z 2022-01-12T14:27:35.898281Z 2022-01-12T14:27:33.769276Z 2022-01-12T14:27:31.099133Z 2022-01-12T14:27:24.333916Z 2022-01-12T14:27:23.035455Z 2022-01-12T14:27:24.395099Z 2022-01-12T14:27:30.787554Z 2022-01-12T14:27:24.281516Z 2022-01-12T14:27:20.703305Z 2022-01-12T14:27:14.613150Z 2022-01-12T14:27:22.221124Z 2022-01-12T14:27:14.236367Z 2022-01-12T14:27:15.234302Z 2022-01-12T14:33:39.470321Z 2022-01-12T14:33:25.551931Z 2022-01-12T14:33:37.798426Z 2022-01-12T14:33:27.458466Z 2022-01-12T14:33:30.344268Z 2022-01-12T14:33:22.277141Z 2022-01-12T14:33:26.650019Z 2022-01-12T14:33:23.188433Z 2022-01-12T14:33:34.677289Z 2022-01-12T14:33:23.031459Z 2022-01-12T14:33:37.142710Z 2022-01-12T14:33:24.667686Z 2022-01-12T14:33:37.166066Z 2022-01-12T14:33:21.407220Z 2022-01-12T14:33:33.790360Z 2022-01-12T14:33:21.681061Z 2022-01-12T14:33:33.274389Z 2022-01-12T14:33:23.022673Z 2022-01-12T14:33:34.770192Z 2022-01-12T14:33:25.351019Z 2022-01-12T14:33:34.240363Z 2022-01-12T14:33:24.368451Z 2022-01-12T14:33:32.718067Z 2022-01-12T14:33:24.071073Z 2022-01-12T14:33:35.570464Z 2022-01-12T14:33:22.173217Z 2022-01-12T14:33:32.830121Z 2022-01-12T14:33:18.139580Z 2022-01-12T14:33:33.012198Z 2022-01-12T14:33:16.901334Z 2022-01-12T14:33:33.226186Z 2022-01-12T14:33:16.896982Z 2022-01-12T14:33:27.116805Z 2022-01-12T14:33:15.337488Z 2022-01-12T14:33:26.449143Z 2022-01-12T14:33:13.980844Z 2022-01-12T14:33:30.249385Z 2022-01-12T14:33:17.968146Z 2022-01-12T14:33:30.115109Z 2022-01-12T14:33:13.660245Z 2022-01-12T14:33:26.971119Z 2022-01-12T14:33:17.958826Z 2022-01-12T14:33:25.038130Z 2022-01-12T14:33:14.075954Z 2022-01-12T14:33:22.090188Z 2022-01-12T14:33:12.514984Z 2022-01-12T14:33:23.874288Z 2022-01-12T14:33:11.237980Z 2022-01-12T14:33:23.252124Z 2022-01-12T14:33:13.608476Z 2022-01-12T14:33:16.815452Z 2022-01-12T14:33:06.346180Z 2022-01-12T14:33:20.824275Z 2022-01-12T14:33:14.001115Z 2022-01-12T14:33:10.577155Z 2022-01-12T14:33:10.268071Z 2022-01-12T14:33:19.627458Z 2022-01-12T14:33:04.994659Z 2022-01-12T14:33:21.184105Z 2022-01-12T14:33:10.385028Z 2022-01-12T14:33:08.085825Z 2022-01-12T14:33:05.408628Z 2022-01-12T14:33:19.712143Z 2022-01-12T14:32:59.224995Z 2022-01-12T14:33:06.743039Z 2022-01-12T14:32:56.136989Z 2022-01-12T14:33:02.652179Z 2022-01-12T14:32:58.974999Z 2022-01-12T14:33:10.086064Z 2022-01-12T14:33:01.169026Z 2022-01-12T14:33:04.401228Z 2022-01-12T14:32:39.645120Z 2022-01-12T14:32:51.225324Z 2022-01-12T14:32:52.294992Z 2022-01-12T14:32:12.254271Z 2022-01-12T14:32:19.536162Z 2022-01-12T14:32:19.353044Z 2022-01-12T14:32:44.625127Z 2022-01-12T14:32:13.603269Z 2022-01-12T14:32:36.570139Z 2022-01-12T14:32:12.890070Z 2022-01-12T14:32:24.894285Z 2022-01-12T14:32:11.060937Z 2022-01-12T14:32:35.945055Z 2022-01-12T14:32:06.568375Z 2022-01-12T14:32:22.565056Z 2022-01-12T14:32:07.301531Z 2022-01-12T14:32:27.491498Z 2022-01-12T14:32:00.055135Z 2022-01-12T14:32:26.190067Z 2022-01-12T14:31:56.895076Z 2022-01-12T14:32:08.921342Z 2022-01-12T14:31:55.182967Z 2022-01-12T14:32:05.060544Z 2022-01-12T14:31:47.112031Z 2022-01-12T14:31:52.258139Z 2022-01-12T14:31:44.217915Z 2022-01-12T14:31:47.797301Z 2022-01-12T14:31:42.778392Z 2022-01-12T14:31:45.269066Z 2022-01-12T14:37:58.994817Z 2022-01-12T14:38:08.316404Z 2022-01-12T14:37:58.175475Z 2022-01-12T14:38:09.913238Z 2022-01-12T14:37:57.814794Z 2022-01-12T14:38:08.796112Z 2022-01-12T14:37:57.814794Z 2022-01-12T14:38:08.378231Z 2022-01-12T14:37:58.411621Z 2022-01-12T14:38:09.367740Z 2022-01-12T14:37:57.435117Z 2022-01-12T14:38:07.987283Z 2022-01-12T14:37:56.447709Z 2022-01-12T14:38:07.384013Z 2022-01-12T14:37:56.172862Z 2022-01-12T14:38:11.426320Z 2022-01-12T14:37:54.952157Z 2022-01-12T14:38:03.688821Z 2022-01-12T14:37:53.621964Z 2022-01-12T14:38:00.520069Z 2022-01-12T14:37:54.354579Z 2022-01-12T14:38:02.949378Z 2022-01-12T14:37:47.159269Z 2022-01-12T14:38:02.403047Z 2022-01-12T14:37:55.555166Z 2022-01-12T14:38:05.699639Z 2022-01-12T14:37:48.410806Z 2022-01-12T14:38:05.020275Z 2022-01-12T14:37:51.037712Z 2022-01-12T14:38:05.939959Z 2022-01-12T14:37:48.793167Z 2022-01-12T14:38:04.679532Z 2022-01-12T14:37:44.536053Z 2022-01-12T14:37:58.372933Z 2022-01-12T14:37:54.748337Z 2022-01-12T14:38:04.655340Z 2022-01-12T14:37:47.549501Z 2022-01-12T14:38:00.504202Z 2022-01-12T14:37:53.152386Z 2022-01-12T14:38:03.911138Z 2022-01-12T14:37:49.675350Z 2022-01-12T14:38:04.079097Z 2022-01-12T14:37:46.411713Z 2022-01-12T14:37:48.690945Z 2022-01-12T14:37:59.236328Z 2022-01-12T14:38:01.000447Z 2022-01-12T14:37:43.791108Z 2022-01-12T14:37:50.252134Z 2022-01-12T14:37:47.405446Z 2022-01-12T14:37:53.662813Z 2022-01-12T14:37:37.738108Z 2022-01-12T14:37:50.252134Z 2022-01-12T14:37:40.634974Z 2022-01-12T14:37:50.787067Z 2022-01-12T14:37:43.274175Z 2022-01-12T14:37:47.261044Z 2022-01-12T14:37:34.851036Z 2022-01-12T14:37:46.511082Z 2022-01-12T14:37:35.202035Z 2022-01-12T14:37:50.059114Z 2022-01-12T14:37:29.229560Z 2022-01-12T14:37:41.750734Z 2022-01-12T14:37:28.230986Z 2022-01-12T14:37:40.027929Z 2022-01-12T14:37:30.366534Z 2022-01-12T14:37:23.505673Z 2022-01-12T14:37:47.773029Z 2022-01-12T14:37:41.001884Z 2022-01-12T14:37:19.480992Z 2022-01-12T14:37:44.428091Z 2022-01-12T14:37:14.432265Z 2022-01-12T14:37:34.032728Z 2022-01-12T14:37:47.323085Z 2022-01-12T14:37:15.142423Z 2022-01-12T14:37:18.651489Z 2022-01-12T14:37:31.268068Z 2022-01-12T14:37:06.743203Z 2022-01-12T14:37:27.316479Z 2022-01-12T14:37:01.694004Z 2022-01-12T14:36:45.893021Z 2022-01-12T14:36:33.706196Z 2022-01-12T14:36:52.984747Z 2022-01-12T14:36:33.706196Z 2022-01-12T14:36:42.701376Z 2022-01-12T14:36:34.510412Z 2022-01-12T14:36:41.197283Z 2022-01-12T14:36:31.473592Z 2022-01-12T14:36:48.771060Z 2022-01-12T14:36:30.933499Z 2022-01-12T14:36:33.514577Z 2022-01-12T14:36:29.655950Z 2022-01-12T14:36:33.450065Z 2022-01-12T14:36:28.512356Z 2022-01-12T14:36:40.015621Z 2022-01-12T14:36:29.007007Z 2022-01-12T14:36:26.354066Z 2022-01-12T14:36:20.392016Z 2022-01-12T14:36:22.030930Z 2022-01-12T14:36:17.836332Z 2022-01-12T14:36:22.582215Z 2022-01-12T14:43:29.465521Z 2022-01-12T14:43:18.055885Z 2022-01-12T14:43:29.123052Z 2022-01-12T14:43:13.048022Z 2022-01-12T14:43:30.498199Z 2022-01-12T14:43:18.984033Z 2022-01-12T14:43:30.161067Z 2022-01-12T14:43:17.944487Z 2022-01-12T14:43:30.352515Z 2022-01-12T14:43:20.633996Z 2022-01-12T14:43:29.938277Z 2022-01-12T14:43:18.525115Z 2022-01-12T14:43:28.511041Z 2022-01-12T14:43:18.715522Z 2022-01-12T14:43:29.994394Z 2022-01-12T14:43:20.341544Z 2022-01-12T14:43:18.214046Z 2022-01-12T14:43:14.613241Z 2022-01-12T14:43:25.926835Z 2022-01-12T14:43:19.117459Z 2022-01-12T14:43:31.599156Z 2022-01-12T14:43:13.746363Z 2022-01-12T14:43:27.966060Z 2022-01-12T14:43:15.303504Z 2022-01-12T14:43:25.760547Z 2022-01-12T14:43:11.578278Z 2022-01-12T14:43:23.912134Z 2022-01-12T14:43:12.616073Z 2022-01-12T14:43:22.579620Z 2022-01-12T14:43:14.007473Z 2022-01-12T14:43:17.602156Z 2022-01-12T14:43:12.763386Z 2022-01-12T14:43:23.635240Z 2022-01-12T14:43:17.188063Z 2022-01-12T14:43:08.018713Z 2022-01-12T14:43:22.050785Z 2022-01-12T14:43:25.126062Z 2022-01-12T14:43:11.515235Z 2022-01-12T14:43:30.025505Z 2022-01-12T14:43:12.285056Z 2022-01-12T14:43:21.070386Z 2022-01-12T14:43:16.799223Z 2022-01-12T14:43:01.293701Z 2022-01-12T14:43:14.752271Z 2022-01-12T14:43:14.939544Z 2022-01-12T14:43:00.037350Z 2022-01-12T14:43:10.955199Z 2022-01-12T14:42:59.391105Z 2022-01-12T14:43:09.586948Z 2022-01-12T14:43:03.470283Z 2022-01-12T14:42:54.029761Z 2022-01-12T14:43:08.271051Z 2022-01-12T14:43:00.940017Z 2022-01-12T14:43:03.979187Z 2022-01-12T14:42:52.052079Z 2022-01-12T14:43:08.744085Z 2022-01-12T14:42:58.284267Z 2022-01-12T14:43:09.480115Z 2022-01-12T14:42:51.551027Z 2022-01-12T14:43:01.513123Z 2022-01-12T14:42:48.740092Z 2022-01-12T14:43:02.922246Z 2022-01-12T14:42:55.012662Z 2022-01-12T14:42:50.877160Z 2022-01-12T14:42:41.702008Z 2022-01-12T14:42:53.257680Z 2022-01-12T14:42:47.342461Z 2022-01-12T14:42:53.603094Z 2022-01-12T14:42:20.595108Z 2022-01-12T14:42:47.355274Z 2022-01-12T14:42:42.648026Z 2022-01-12T14:42:58.721137Z 2022-01-12T14:42:39.847089Z 2022-01-12T14:42:45.796853Z 2022-01-12T14:42:28.755984Z 2022-01-12T14:42:55.732009Z 2022-01-12T14:42:25.473036Z 2022-01-12T14:42:39.181982Z 2022-01-12T14:42:35.930871Z 2022-01-12T14:42:33.662690Z 2022-01-12T14:42:41.179181Z 2022-01-12T14:42:42.388065Z 2022-01-12T14:42:28.638087Z 2022-01-12T14:42:27.929392Z 2022-01-12T14:42:27.481130Z 2022-01-12T14:42:18.168042Z 2022-01-12T14:42:15.650838Z 2022-01-12T14:42:31.550047Z 2022-01-12T14:42:23.678030Z 2022-01-12T14:42:25.494229Z 2022-01-12T14:41:28.646087Z 2022-01-12T14:41:42.966067Z 2022-01-12T14:41:32.016719Z 2022-01-12T14:41:27.155088Z 2022-01-12T14:41:27.179022Z 2022-01-12T14:41:38.042990Z 2022-01-12T14:41:30.273065Z 2022-01-12T14:41:29.629205Z 2022-01-12T14:41:13.818042Z 2022-01-12T14:41:11.574840Z", + "failed_job_ids": " []", + "finished": "2022-01-12T14:14:49.459310Z 2022-01-12T14:15:37.049038Z 2022-01-12T14:15:31.301908Z 2022-01-12T14:14:42.421943Z 2022-01-12T14:14:39.397586Z 2022-01-12T14:15:23.199601Z 2022-01-12T14:14:40.582447Z 2022-01-12T14:15:40.183634Z 2022-01-12T14:14:38.876098Z 2022-01-12T14:15:25.041986Z 2022-01-12T14:15:20.252910Z 2022-01-12T14:15:07.524307Z 2022-01-12T14:17:00.753535Z 2022-01-12T14:15:07.227265Z 2022-01-12T14:16:00.229931Z 2022-01-12T14:15:09.265833Z 2022-01-12T14:17:09.803033Z 2022-01-12T14:15:08.458026Z 2022-01-12T14:15:51.407801Z 2022-01-12T14:15:03.855386Z 2022-01-12T14:16:31.005628Z 2022-01-12T14:14:57.456985Z 2022-01-12T14:17:19.992652Z 2022-01-12T14:14:59.304997Z 2022-01-12T14:16:51.643367Z 2022-01-12T14:14:48.443108Z 2022-01-12T14:16:20.356041Z 2022-01-12T14:14:24.332957Z 2022-01-12T14:15:42.778332Z 2022-01-12T14:14:19.124960Z 2022-01-12T14:16:10.747260Z 2022-01-12T14:14:27.074590Z 2022-01-12T14:16:41.689189Z 2022-01-12T14:14:12.592078Z 2022-01-12T14:14:36.506971Z 2022-01-12T14:14:13.610917Z 2022-01-12T14:15:56.061866Z 2022-01-12T14:15:36.507474Z 2022-01-12T14:15:55.174333Z 2022-01-12T14:15:37.934132Z 2022-01-12T14:15:39.026708Z 2022-01-12T14:15:36.137009Z 2022-01-12T14:15:37.928988Z 2022-01-12T14:15:35.518982Z 2022-01-12T14:15:38.713279Z 2022-01-12T14:15:36.456037Z 2022-01-12T14:15:50.134250Z 2022-01-12T14:15:31.725632Z 2022-01-12T14:15:54.824083Z 2022-01-12T14:15:38.808789Z 2022-01-12T14:15:24.970144Z 2022-01-12T14:15:51.776302Z 2022-01-12T14:15:32.750712Z 2022-01-12T14:15:52.517135Z 2022-01-12T14:15:31.834984Z 2022-01-12T14:15:54.800675Z 2022-01-12T14:15:30.844872Z 2022-01-12T14:15:50.807133Z 2022-01-12T14:15:27.270920Z 2022-01-12T14:15:49.176911Z 2022-01-12T14:15:29.322103Z 2022-01-12T14:15:55.152692Z 2022-01-12T14:15:23.287274Z 2022-01-12T14:15:52.474310Z 2022-01-12T14:15:28.475942Z 2022-01-12T14:15:54.022873Z 2022-01-12T14:15:20.721295Z 2022-01-12T14:15:50.468540Z 2022-01-12T14:15:22.619183Z 2022-01-12T14:15:50.862280Z 2022-01-12T14:15:20.961880Z 2022-01-12T14:15:49.421201Z 2022-01-12T14:15:53.578438Z 2022-01-12T14:15:49.683634Z 2022-01-12T14:14:55.058297Z 2022-01-12T14:15:48.093966Z 2022-01-12T14:14:58.750992Z 2022-01-12T14:15:44.267420Z 2022-01-12T14:14:55.470419Z 2022-01-12T14:15:48.936476Z 2022-01-12T14:15:02.296753Z 2022-01-12T14:15:43.863719Z 2022-01-12T14:14:59.534322Z 2022-01-12T14:15:46.066105Z 2022-01-12T14:14:53.783048Z 2022-01-12T14:15:41.835234Z 2022-01-12T14:14:56.491589Z 2022-01-12T14:15:34.487741Z 2022-01-12T14:14:48.919830Z 2022-01-12T14:15:37.074450Z 2022-01-12T14:14:51.952028Z 2022-01-12T14:15:40.990639Z 2022-01-12T14:14:50.575289Z 2022-01-12T14:15:44.356945Z 2022-01-12T14:14:48.852757Z 2022-01-12T14:15:25.867176Z 2022-01-12T14:14:47.122979Z 2022-01-12T14:15:42.604114Z 2022-01-12T14:14:43.445153Z 2022-01-12T14:15:32.264009Z 2022-01-12T14:29:12.507986Z 2022-01-12T14:29:03.372696Z 2022-01-12T14:29:10.837188Z 2022-01-12T14:29:04.057936Z 2022-01-12T14:29:10.224090Z 2022-01-12T14:29:01.170045Z 2022-01-12T14:29:12.426303Z 2022-01-12T14:29:02.803880Z 2022-01-12T14:29:12.054939Z 2022-01-12T14:29:02.589301Z 2022-01-12T14:29:11.784992Z 2022-01-12T14:29:03.011232Z 2022-01-12T14:29:11.512081Z 2022-01-12T14:29:03.364952Z 2022-01-12T14:29:10.150059Z 2022-01-12T14:29:01.844979Z 2022-01-12T14:29:12.021353Z 2022-01-12T14:28:58.726086Z 2022-01-12T14:29:07.429197Z 2022-01-12T14:28:55.188995Z 2022-01-12T14:29:11.314024Z 2022-01-12T14:29:00.813987Z 2022-01-12T14:29:03.382133Z 2022-01-12T14:29:04.356972Z 2022-01-12T14:29:00.545506Z 2022-01-12T14:29:02.742281Z 2022-01-12T14:28:54.434529Z 2022-01-12T14:29:04.517627Z 2022-01-12T14:28:53.814179Z 2022-01-12T14:29:01.285875Z 2022-01-12T14:28:50.682063Z 2022-01-12T14:29:08.174831Z 2022-01-12T14:29:04.849286Z 2022-01-12T14:29:02.044140Z 2022-01-12T14:29:09.211238Z 2022-01-12T14:29:02.390773Z 2022-01-12T14:28:56.152287Z 2022-01-12T14:29:03.970449Z 2022-01-12T14:28:52.285267Z 2022-01-12T14:29:02.735307Z 2022-01-12T14:28:54.739456Z 2022-01-12T14:28:57.791300Z 2022-01-12T14:28:55.929492Z 2022-01-12T14:28:54.677171Z 2022-01-12T14:28:57.742101Z 2022-01-12T14:28:58.841879Z 2022-01-12T14:28:41.090957Z 2022-01-12T14:28:56.187822Z 2022-01-12T14:28:53.852338Z 2022-01-12T14:28:53.399868Z 2022-01-12T14:28:46.663042Z 2022-01-12T14:28:51.279083Z 2022-01-12T14:28:55.368999Z 2022-01-12T14:28:53.132219Z 2022-01-12T14:28:44.832107Z 2022-01-12T14:28:57.154585Z 2022-01-12T14:28:46.947681Z 2022-01-12T14:28:49.652417Z 2022-01-12T14:28:49.331947Z 2022-01-12T14:28:41.971445Z 2022-01-12T14:28:35.173320Z 2022-01-12T14:28:52.005433Z 2022-01-12T14:28:43.659669Z 2022-01-12T14:28:43.195808Z 2022-01-12T14:28:41.513054Z 2022-01-12T14:28:36.472523Z 2022-01-12T14:28:43.467975Z 2022-01-12T14:28:51.439123Z 2022-01-12T14:28:27.771948Z 2022-01-12T14:28:29.046856Z 2022-01-12T14:28:27.785483Z 2022-01-12T14:28:26.489668Z 2022-01-12T14:28:34.888254Z 2022-01-12T14:28:27.156789Z 2022-01-12T14:28:39.443612Z 2022-01-12T14:28:17.563629Z 2022-01-12T14:27:59.771616Z 2022-01-12T14:28:04.914348Z 2022-01-12T14:27:55.470976Z 2022-01-12T14:28:06.463125Z 2022-01-12T14:27:52.814986Z 2022-01-12T14:27:59.239198Z 2022-01-12T14:27:49.345338Z 2022-01-12T14:27:58.535483Z 2022-01-12T14:27:39.570088Z 2022-01-12T14:27:47.098675Z 2022-01-12T14:27:51.684080Z 2022-01-12T14:27:39.450642Z 2022-01-12T14:27:39.262401Z 2022-01-12T14:27:35.768839Z 2022-01-12T14:27:28.896155Z 2022-01-12T14:27:27.089492Z 2022-01-12T14:27:28.240797Z 2022-01-12T14:27:35.057804Z 2022-01-12T14:27:28.993154Z 2022-01-12T14:27:25.638800Z 2022-01-12T14:27:18.324218Z 2022-01-12T14:27:26.138175Z 2022-01-12T14:27:18.294199Z 2022-01-12T14:27:19.846925Z 2022-01-12T14:33:42.270689Z 2022-01-12T14:33:29.993265Z 2022-01-12T14:33:42.114330Z 2022-01-12T14:33:29.540305Z 2022-01-12T14:33:37.319915Z 2022-01-12T14:33:26.869440Z 2022-01-12T14:33:32.554951Z 2022-01-12T14:33:28.004175Z 2022-01-12T14:33:40.020910Z 2022-01-12T14:33:27.787625Z 2022-01-12T14:33:41.880339Z 2022-01-12T14:33:29.260470Z 2022-01-12T14:33:41.534077Z 2022-01-12T14:33:25.427965Z 2022-01-12T14:33:37.986630Z 2022-01-12T14:33:26.705310Z 2022-01-12T14:33:40.347182Z 2022-01-12T14:33:29.195553Z 2022-01-12T14:33:40.180026Z 2022-01-12T14:33:29.912967Z 2022-01-12T14:33:40.373692Z 2022-01-12T14:33:28.853476Z 2022-01-12T14:33:37.541593Z 2022-01-12T14:33:28.656235Z 2022-01-12T14:33:40.385205Z 2022-01-12T14:33:26.582967Z 2022-01-12T14:33:39.363547Z 2022-01-12T14:33:22.826857Z 2022-01-12T14:33:40.273606Z 2022-01-12T14:33:23.450592Z 2022-01-12T14:33:40.442648Z 2022-01-12T14:33:23.454753Z 2022-01-12T14:33:34.172148Z 2022-01-12T14:33:21.001925Z 2022-01-12T14:33:31.172634Z 2022-01-12T14:33:18.965993Z 2022-01-12T14:33:35.245235Z 2022-01-12T14:33:23.856351Z 2022-01-12T14:33:35.216016Z 2022-01-12T14:33:19.926463Z 2022-01-12T14:33:34.079412Z 2022-01-12T14:33:23.060027Z 2022-01-12T14:33:32.365966Z 2022-01-12T14:33:18.547958Z 2022-01-12T14:33:29.025113Z 2022-01-12T14:33:17.595292Z 2022-01-12T14:33:29.890933Z 2022-01-12T14:33:16.811020Z 2022-01-12T14:33:29.871838Z 2022-01-12T14:33:19.455708Z 2022-01-12T14:33:24.870440Z 2022-01-12T14:33:11.837090Z 2022-01-12T14:33:27.441584Z 2022-01-12T14:33:19.180711Z 2022-01-12T14:33:16.326903Z 2022-01-12T14:33:15.718967Z 2022-01-12T14:33:26.494356Z 2022-01-12T14:33:09.923629Z 2022-01-12T14:33:26.768215Z 2022-01-12T14:33:17.715029Z 2022-01-12T14:33:15.272096Z 2022-01-12T14:33:12.971961Z 2022-01-12T14:33:27.755710Z 2022-01-12T14:33:04.294976Z 2022-01-12T14:33:13.468973Z 2022-01-12T14:33:01.332091Z 2022-01-12T14:33:08.752618Z 2022-01-12T14:33:04.852217Z 2022-01-12T14:33:16.157434Z 2022-01-12T14:33:07.896954Z 2022-01-12T14:33:10.045284Z 2022-01-12T14:32:46.289424Z 2022-01-12T14:32:57.820521Z 2022-01-12T14:32:56.935825Z 2022-01-12T14:32:17.378356Z 2022-01-12T14:32:24.804696Z 2022-01-12T14:32:24.727074Z 2022-01-12T14:32:49.739513Z 2022-01-12T14:32:19.205317Z 2022-01-12T14:32:44.517915Z 2022-01-12T14:32:18.394815Z 2022-01-12T14:32:32.933125Z 2022-01-12T14:32:15.203164Z 2022-01-12T14:32:41.522966Z 2022-01-12T14:32:11.281097Z 2022-01-12T14:32:28.584241Z 2022-01-12T14:32:12.977179Z 2022-01-12T14:32:35.288351Z 2022-01-12T14:32:04.173034Z 2022-01-12T14:32:31.443853Z 2022-01-12T14:32:01.118970Z 2022-01-12T14:32:15.824566Z 2022-01-12T14:31:59.642156Z 2022-01-12T14:32:09.332015Z 2022-01-12T14:31:50.966466Z 2022-01-12T14:31:54.277954Z 2022-01-12T14:31:48.082983Z 2022-01-12T14:31:50.102138Z 2022-01-12T14:31:46.629105Z 2022-01-12T14:31:49.326234Z 2022-01-12T14:38:02.504339Z 2022-01-12T14:38:12.888632Z 2022-01-12T14:38:02.102426Z 2022-01-12T14:38:14.205859Z 2022-01-12T14:38:01.821720Z 2022-01-12T14:38:13.042159Z 2022-01-12T14:38:02.625801Z 2022-01-12T14:38:12.915747Z 2022-01-12T14:38:01.847654Z 2022-01-12T14:38:14.016304Z 2022-01-12T14:38:01.912296Z 2022-01-12T14:38:12.424381Z 2022-01-12T14:38:01.082100Z 2022-01-12T14:38:11.379424Z 2022-01-12T14:38:00.557599Z 2022-01-12T14:38:13.884815Z 2022-01-12T14:37:59.691614Z 2022-01-12T14:38:10.778147Z 2022-01-12T14:37:57.545511Z 2022-01-12T14:38:07.330784Z 2022-01-12T14:38:00.453243Z 2022-01-12T14:38:09.778988Z 2022-01-12T14:37:52.046328Z 2022-01-12T14:38:08.552059Z 2022-01-12T14:38:00.421215Z 2022-01-12T14:38:10.333698Z 2022-01-12T14:37:53.110393Z 2022-01-12T14:38:10.319947Z 2022-01-12T14:37:56.210398Z 2022-01-12T14:38:10.488058Z 2022-01-12T14:37:54.699343Z 2022-01-12T14:38:10.198864Z 2022-01-12T14:37:50.741158Z 2022-01-12T14:38:04.604298Z 2022-01-12T14:38:00.197419Z 2022-01-12T14:38:10.000581Z 2022-01-12T14:37:53.110109Z 2022-01-12T14:38:05.038306Z 2022-01-12T14:37:58.677461Z 2022-01-12T14:38:08.933997Z 2022-01-12T14:37:54.230475Z 2022-01-12T14:38:08.178538Z 2022-01-12T14:37:53.346908Z 2022-01-12T14:37:53.837309Z 2022-01-12T14:38:04.516840Z 2022-01-12T14:38:06.103041Z 2022-01-12T14:37:49.236006Z 2022-01-12T14:37:56.151880Z 2022-01-12T14:37:53.190011Z 2022-01-12T14:37:58.725123Z 2022-01-12T14:37:43.886083Z 2022-01-12T14:37:55.916776Z 2022-01-12T14:37:45.746361Z 2022-01-12T14:37:58.300260Z 2022-01-12T14:37:47.731323Z 2022-01-12T14:37:55.014886Z 2022-01-12T14:37:41.848982Z 2022-01-12T14:37:52.885858Z 2022-01-12T14:37:41.581104Z 2022-01-12T14:37:56.539171Z 2022-01-12T14:37:35.569310Z 2022-01-12T14:37:48.386422Z 2022-01-12T14:37:35.787163Z 2022-01-12T14:37:46.314092Z 2022-01-12T14:37:36.287110Z 2022-01-12T14:37:29.697936Z 2022-01-12T14:37:55.550640Z 2022-01-12T14:37:49.039075Z 2022-01-12T14:37:25.393795Z 2022-01-12T14:37:52.656919Z 2022-01-12T14:37:20.117909Z 2022-01-12T14:37:41.090193Z 2022-01-12T14:37:53.989256Z 2022-01-12T14:37:21.355761Z 2022-01-12T14:37:24.035720Z 2022-01-12T14:37:39.842055Z 2022-01-12T14:37:14.113090Z 2022-01-12T14:37:31.865113Z 2022-01-12T14:37:06.809391Z 2022-01-12T14:36:52.788715Z 2022-01-12T14:36:38.136101Z 2022-01-12T14:36:59.281215Z 2022-01-12T14:36:37.995184Z 2022-01-12T14:36:48.208705Z 2022-01-12T14:36:38.349817Z 2022-01-12T14:36:46.576220Z 2022-01-12T14:36:35.485250Z 2022-01-12T14:36:54.049319Z 2022-01-12T14:36:34.936065Z 2022-01-12T14:36:38.445568Z 2022-01-12T14:36:34.362999Z 2022-01-12T14:36:38.448072Z 2022-01-12T14:36:32.800065Z 2022-01-12T14:36:44.453530Z 2022-01-12T14:36:34.313870Z 2022-01-12T14:36:31.042034Z 2022-01-12T14:36:24.218632Z 2022-01-12T14:36:26.605074Z 2022-01-12T14:36:22.448217Z 2022-01-12T14:36:26.440196Z 2022-01-12T14:43:34.296272Z 2022-01-12T14:43:22.475691Z 2022-01-12T14:43:34.554364Z 2022-01-12T14:43:18.226907Z 2022-01-12T14:43:34.973925Z 2022-01-12T14:43:22.781295Z 2022-01-12T14:43:35.256525Z 2022-01-12T14:43:22.632027Z 2022-01-12T14:43:35.223055Z 2022-01-12T14:43:24.789774Z 2022-01-12T14:43:34.558399Z 2022-01-12T14:43:23.122811Z 2022-01-12T14:43:33.059341Z 2022-01-12T14:43:23.794954Z 2022-01-12T14:43:34.621912Z 2022-01-12T14:43:24.309589Z 2022-01-12T14:43:23.910218Z 2022-01-12T14:43:20.742159Z 2022-01-12T14:43:30.915095Z 2022-01-12T14:43:23.521025Z 2022-01-12T14:43:35.389036Z 2022-01-12T14:43:19.272986Z 2022-01-12T14:43:32.479795Z 2022-01-12T14:43:21.085657Z 2022-01-12T14:43:31.205728Z 2022-01-12T14:43:17.878590Z 2022-01-12T14:43:28.330293Z 2022-01-12T14:43:18.020315Z 2022-01-12T14:43:27.871702Z 2022-01-12T14:43:19.316132Z 2022-01-12T14:43:22.955888Z 2022-01-12T14:43:18.947522Z 2022-01-12T14:43:30.362853Z 2022-01-12T14:43:22.007292Z 2022-01-12T14:43:14.697659Z 2022-01-12T14:43:28.850100Z 2022-01-12T14:43:30.721156Z 2022-01-12T14:43:17.872648Z 2022-01-12T14:43:35.171624Z 2022-01-12T14:43:18.723209Z 2022-01-12T14:43:25.731130Z 2022-01-12T14:43:24.034631Z 2022-01-12T14:43:07.316916Z 2022-01-12T14:43:19.792016Z 2022-01-12T14:43:22.600126Z 2022-01-12T14:43:07.319652Z 2022-01-12T14:43:20.988385Z 2022-01-12T14:43:07.490798Z 2022-01-12T14:43:16.723528Z 2022-01-12T14:43:11.968825Z 2022-01-12T14:42:58.046048Z 2022-01-12T14:43:15.247661Z 2022-01-12T14:43:07.725038Z 2022-01-12T14:43:12.757913Z 2022-01-12T14:42:57.895267Z 2022-01-12T14:43:15.337156Z 2022-01-12T14:43:04.737965Z 2022-01-12T14:43:16.983025Z 2022-01-12T14:42:58.206111Z 2022-01-12T14:43:11.564981Z 2022-01-12T14:42:55.861975Z 2022-01-12T14:43:10.461193Z 2022-01-12T14:43:01.191003Z 2022-01-12T14:42:57.985373Z 2022-01-12T14:42:48.997877Z 2022-01-12T14:43:03.869953Z 2022-01-12T14:42:52.501963Z 2022-01-12T14:43:00.635237Z 2022-01-12T14:42:26.969083Z 2022-01-12T14:42:56.023770Z 2022-01-12T14:42:49.365919Z 2022-01-12T14:43:05.733235Z 2022-01-12T14:42:45.772343Z 2022-01-12T14:42:53.092657Z 2022-01-12T14:42:34.116492Z 2022-01-12T14:43:02.919842Z 2022-01-12T14:42:32.456237Z 2022-01-12T14:42:46.233166Z 2022-01-12T14:42:42.896526Z 2022-01-12T14:42:42.183093Z 2022-01-12T14:42:47.399034Z 2022-01-12T14:42:49.639285Z 2022-01-12T14:42:35.618267Z 2022-01-12T14:42:34.878232Z 2022-01-12T14:42:33.838801Z 2022-01-12T14:42:26.080676Z 2022-01-12T14:42:23.196018Z 2022-01-12T14:42:39.484216Z 2022-01-12T14:42:30.370912Z 2022-01-12T14:42:32.970310Z 2022-01-12T14:41:34.804571Z 2022-01-12T14:41:47.693101Z 2022-01-12T14:41:37.556088Z 2022-01-12T14:41:31.671062Z 2022-01-12T14:41:32.497575Z 2022-01-12T14:41:43.956982Z 2022-01-12T14:41:35.963938Z 2022-01-12T14:41:34.154379Z 2022-01-12T14:41:17.560769Z 2022-01-12T14:41:15.436421Z", + "finished_diff": 133.7932404, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 196.989449, + "mean": 65.78731501000001, + "min": 15.45399 + }, + "jobs_events_duration": { + "max": 10.545321, + "mean": 3.9752906880000007, + "min": 0.188201 + }, + "jobs_events_lag": { + "max": -1.69085, + "mean": -5.450654277999999, + "min": -10.612273 + }, + "jobs_waiting": { + "max": 50.467697, + "mean": 27.03678091799998, + "min": 0.689913 + }, + "modified": "2022-01-12T14:14:08.316615Z 2022-01-12T14:14:08.279411Z 2022-01-12T14:14:08.201417Z 2022-01-12T14:14:08.241711Z 2022-01-12T14:14:08.163142Z 2022-01-12T14:14:08.111015Z 2022-01-12T14:14:08.070936Z 2022-01-12T14:14:08.031677Z 2022-01-12T14:14:07.961276Z 2022-01-12T14:14:07.896993Z 2022-01-12T14:14:07.821147Z 2022-01-12T14:14:00.730160Z 2022-01-12T14:14:00.617756Z 2022-01-12T14:14:00.520034Z 2022-01-12T14:14:00.417436Z 2022-01-12T14:14:00.313876Z 2022-01-12T14:14:00.188426Z 2022-01-12T14:14:00.030201Z 2022-01-12T14:13:59.903595Z 2022-01-12T14:13:59.717523Z 2022-01-12T14:13:59.647418Z 2022-01-12T14:13:59.582924Z 2022-01-12T14:13:59.514983Z 2022-01-12T14:13:59.404284Z 2022-01-12T14:13:59.309092Z 2022-01-12T14:13:59.164169Z 2022-01-12T14:13:59.056335Z 2022-01-12T14:13:56.514075Z 2022-01-12T14:13:56.437012Z 2022-01-12T14:13:56.375535Z 2022-01-12T14:13:56.310258Z 2022-01-12T14:13:56.250258Z 2022-01-12T14:13:56.171169Z 2022-01-12T14:13:55.546831Z 2022-01-12T14:13:55.505002Z 2022-01-12T14:13:55.465364Z 2022-01-12T14:14:54.270518Z 2022-01-12T14:14:54.140425Z 2022-01-12T14:14:54.009434Z 2022-01-12T14:14:53.870573Z 2022-01-12T14:14:53.639432Z 2022-01-12T14:14:53.491448Z 2022-01-12T14:14:53.324892Z 2022-01-12T14:14:53.141343Z 2022-01-12T14:14:52.885533Z 2022-01-12T14:14:52.690327Z 2022-01-12T14:14:37.812167Z 2022-01-12T14:14:37.706005Z 2022-01-12T14:14:37.605417Z 2022-01-12T14:14:52.569615Z 2022-01-12T14:14:37.491321Z 2022-01-12T14:14:37.378186Z 2022-01-12T14:14:37.246327Z 2022-01-12T14:14:37.158533Z 2022-01-12T14:14:37.100379Z 2022-01-12T14:14:36.947160Z 2022-01-12T14:14:36.825957Z 2022-01-12T14:14:36.678233Z 2022-01-12T14:14:36.505110Z 2022-01-12T14:14:36.260648Z 2022-01-12T14:14:36.050300Z 2022-01-12T14:14:35.792992Z 2022-01-12T14:14:35.445423Z 2022-01-12T14:14:35.190028Z 2022-01-12T14:14:34.687346Z 2022-01-12T14:14:34.232322Z 2022-01-12T14:14:33.755489Z 2022-01-12T14:14:33.491329Z 2022-01-12T14:14:33.196247Z 2022-01-12T14:14:33.086200Z 2022-01-12T14:14:32.899268Z 2022-01-12T14:14:32.711022Z 2022-01-12T14:14:32.383054Z 2022-01-12T14:14:09.307105Z 2022-01-12T14:14:09.273655Z 2022-01-12T14:14:09.238161Z 2022-01-12T14:14:09.204184Z 2022-01-12T14:14:09.170082Z 2022-01-12T14:14:09.135900Z 2022-01-12T14:14:09.100778Z 2022-01-12T14:14:09.064577Z 2022-01-12T14:14:09.027507Z 2022-01-12T14:14:08.992718Z 2022-01-12T14:14:08.956028Z 2022-01-12T14:14:08.921382Z 2022-01-12T14:14:08.883695Z 2022-01-12T14:14:08.847369Z 2022-01-12T14:14:08.809999Z 2022-01-12T14:14:08.772696Z 2022-01-12T14:14:08.737080Z 2022-01-12T14:14:08.698443Z 2022-01-12T14:14:08.661643Z 2022-01-12T14:14:08.625393Z 2022-01-12T14:14:08.590623Z 2022-01-12T14:14:08.554806Z 2022-01-12T14:14:08.517911Z 2022-01-12T14:14:08.483709Z 2022-01-12T14:14:08.448582Z 2022-01-12T14:14:08.408101Z 2022-01-12T14:14:08.361032Z 2022-01-12T14:27:57.384004Z 2022-01-12T14:27:57.263193Z 2022-01-12T14:27:57.132184Z 2022-01-12T14:27:56.986305Z 2022-01-12T14:27:56.823077Z 2022-01-12T14:27:56.667159Z 2022-01-12T14:27:56.514298Z 2022-01-12T14:27:56.324324Z 2022-01-12T14:27:56.166307Z 2022-01-12T14:27:55.993168Z 2022-01-12T14:27:55.843121Z 2022-01-12T14:27:55.689433Z 2022-01-12T14:27:55.543810Z 2022-01-12T14:27:55.432233Z 2022-01-12T14:27:55.276146Z 2022-01-12T14:27:55.176176Z 2022-01-12T14:27:55.043517Z 2022-01-12T14:27:54.931099Z 2022-01-12T14:27:54.873932Z 2022-01-12T14:27:54.823026Z 2022-01-12T14:27:54.724935Z 2022-01-12T14:27:30.398547Z 2022-01-12T14:27:54.608117Z 2022-01-12T14:27:30.355065Z 2022-01-12T14:27:30.299111Z 2022-01-12T14:27:30.250741Z 2022-01-12T14:27:30.178307Z 2022-01-12T14:27:30.079098Z 2022-01-12T14:27:29.976150Z 2022-01-12T14:27:29.846128Z 2022-01-12T14:27:29.724276Z 2022-01-12T14:27:54.476300Z 2022-01-12T14:27:29.556098Z 2022-01-12T14:27:29.368583Z 2022-01-12T14:27:54.332472Z 2022-01-12T14:27:29.219701Z 2022-01-12T14:27:29.139112Z 2022-01-12T14:27:29.062385Z 2022-01-12T14:27:28.963594Z 2022-01-12T14:27:28.893810Z 2022-01-12T14:27:28.840860Z 2022-01-12T14:27:28.780020Z 2022-01-12T14:27:28.708874Z 2022-01-12T14:27:28.660861Z 2022-01-12T14:27:28.602328Z 2022-01-12T14:27:28.497607Z 2022-01-12T14:27:28.430663Z 2022-01-12T14:27:28.362873Z 2022-01-12T14:27:28.297871Z 2022-01-12T14:27:28.229977Z 2022-01-12T14:27:28.163121Z 2022-01-12T14:27:28.081066Z 2022-01-12T14:27:28.013470Z 2022-01-12T14:27:27.965185Z 2022-01-12T14:27:27.904068Z 2022-01-12T14:27:27.853692Z 2022-01-12T14:27:27.775109Z 2022-01-12T14:27:27.626937Z 2022-01-12T14:27:27.515433Z 2022-01-12T14:27:27.336875Z 2022-01-12T14:27:27.227683Z 2022-01-12T14:27:27.090857Z 2022-01-12T14:27:26.919786Z 2022-01-12T14:27:26.783801Z 2022-01-12T14:27:26.682288Z 2022-01-12T14:27:26.591671Z 2022-01-12T14:27:26.501216Z 2022-01-12T14:27:26.398835Z 2022-01-12T14:27:26.144103Z 2022-01-12T14:27:26.003964Z 2022-01-12T14:27:25.817406Z 2022-01-12T14:27:25.660294Z 2022-01-12T14:27:25.483210Z 2022-01-12T14:27:25.277609Z 2022-01-12T14:27:25.035533Z 2022-01-12T14:27:09.031045Z 2022-01-12T14:27:08.828952Z 2022-01-12T14:27:08.548670Z 2022-01-12T14:27:08.334520Z 2022-01-12T14:27:08.017501Z 2022-01-12T14:27:07.838447Z 2022-01-12T14:27:07.578524Z 2022-01-12T14:27:07.169522Z 2022-01-12T14:27:06.876706Z 2022-01-12T14:27:06.572539Z 2022-01-12T14:27:06.397467Z 2022-01-12T14:27:06.208141Z 2022-01-12T14:27:06.008393Z 2022-01-12T14:27:05.868030Z 2022-01-12T14:27:02.967279Z 2022-01-12T14:27:02.914422Z 2022-01-12T14:27:02.823528Z 2022-01-12T14:27:02.782634Z 2022-01-12T14:27:02.734056Z 2022-01-12T14:27:02.646396Z 2022-01-12T14:27:01.350699Z 2022-01-12T14:27:01.281700Z 2022-01-12T14:27:01.240229Z 2022-01-12T14:27:00.650679Z 2022-01-12T14:27:00.611400Z 2022-01-12T14:31:57.996921Z 2022-01-12T14:31:57.937663Z 2022-01-12T14:31:57.892666Z 2022-01-12T14:31:57.834156Z 2022-01-12T14:31:57.783647Z 2022-01-12T14:31:57.746925Z 2022-01-12T14:31:57.710626Z 2022-01-12T14:31:57.673282Z 2022-01-12T14:31:57.637855Z 2022-01-12T14:31:57.599864Z 2022-01-12T14:31:57.554203Z 2022-01-12T14:31:57.493942Z 2022-01-12T14:31:57.431854Z 2022-01-12T14:31:57.367959Z 2022-01-12T14:31:57.308050Z 2022-01-12T14:31:57.247165Z 2022-01-12T14:31:57.195913Z 2022-01-12T14:31:57.144088Z 2022-01-12T14:31:57.075127Z 2022-01-12T14:31:56.999953Z 2022-01-12T14:31:56.919516Z 2022-01-12T14:31:56.796910Z 2022-01-12T14:31:56.718948Z 2022-01-12T14:31:56.638877Z 2022-01-12T14:31:56.558160Z 2022-01-12T14:31:56.457203Z 2022-01-12T14:31:56.381878Z 2022-01-12T14:31:56.330496Z 2022-01-12T14:31:56.260127Z 2022-01-12T14:31:56.156036Z 2022-01-12T14:31:56.054183Z 2022-01-12T14:31:55.931504Z 2022-01-12T14:31:55.864090Z 2022-01-12T14:31:55.797901Z 2022-01-12T14:31:55.686527Z 2022-01-12T14:31:55.569350Z 2022-01-12T14:31:55.475103Z 2022-01-12T14:31:55.395184Z 2022-01-12T14:31:55.301111Z 2022-01-12T14:31:55.232420Z 2022-01-12T14:31:55.170627Z 2022-01-12T14:31:55.115179Z 2022-01-12T14:31:55.068007Z 2022-01-12T14:31:55.019322Z 2022-01-12T14:31:54.979934Z 2022-01-12T14:31:54.942034Z 2022-01-12T14:31:54.875033Z 2022-01-12T14:31:54.801988Z 2022-01-12T14:31:54.731026Z 2022-01-12T14:31:54.635423Z 2022-01-12T14:31:54.552010Z 2022-01-12T14:31:54.471115Z 2022-01-12T14:31:54.350888Z 2022-01-12T14:31:54.267213Z 2022-01-12T14:31:54.189175Z 2022-01-12T14:31:54.099931Z 2022-01-12T14:31:54.018195Z 2022-01-12T14:31:53.937201Z 2022-01-12T14:31:53.829276Z 2022-01-12T14:31:53.697094Z 2022-01-12T14:31:53.563145Z 2022-01-12T14:31:53.517643Z 2022-01-12T14:31:53.476036Z 2022-01-12T14:31:53.429610Z 2022-01-12T14:31:53.390055Z 2022-01-12T14:31:53.344536Z 2022-01-12T14:31:53.300308Z 2022-01-12T14:31:53.256092Z 2022-01-12T14:31:53.215013Z 2022-01-12T14:31:53.162925Z 2022-01-12T14:31:53.118060Z 2022-01-12T14:31:39.919230Z 2022-01-12T14:31:53.047386Z 2022-01-12T14:31:52.938805Z 2022-01-12T14:31:39.440419Z 2022-01-12T14:31:38.956149Z 2022-01-12T14:31:38.589222Z 2022-01-12T14:31:38.227710Z 2022-01-12T14:31:37.857108Z 2022-01-12T14:31:37.512384Z 2022-01-12T14:31:37.287728Z 2022-01-12T14:31:37.175180Z 2022-01-12T14:31:37.063502Z 2022-01-12T14:31:36.813640Z 2022-01-12T14:31:36.657957Z 2022-01-12T14:31:36.486792Z 2022-01-12T14:31:36.159861Z 2022-01-12T14:31:35.920464Z 2022-01-12T14:31:35.807676Z 2022-01-12T14:31:35.647840Z 2022-01-12T14:31:35.496345Z 2022-01-12T14:31:35.356728Z 2022-01-12T14:31:35.200662Z 2022-01-12T14:31:34.861192Z 2022-01-12T14:31:31.353000Z 2022-01-12T14:31:31.200971Z 2022-01-12T14:31:30.921362Z 2022-01-12T14:31:29.946121Z 2022-01-12T14:31:29.903382Z 2022-01-12T14:31:29.848580Z 2022-01-12T14:37:08.316545Z 2022-01-12T14:36:29.164981Z 2022-01-12T14:36:29.044957Z 2022-01-12T14:36:28.918895Z 2022-01-12T14:36:28.793229Z 2022-01-12T14:36:28.668011Z 2022-01-12T14:36:28.526258Z 2022-01-12T14:36:28.475816Z 2022-01-12T14:36:28.433142Z 2022-01-12T14:36:28.393899Z 2022-01-12T14:36:28.349396Z 2022-01-12T14:36:28.302458Z 2022-01-12T14:36:28.253931Z 2022-01-12T14:36:28.206354Z 2022-01-12T14:36:28.151886Z 2022-01-12T14:36:28.099304Z 2022-01-12T14:36:28.048338Z 2022-01-12T14:36:28.006238Z 2022-01-12T14:36:27.957351Z 2022-01-12T14:36:27.918637Z 2022-01-12T14:36:27.881019Z 2022-01-12T14:36:27.838201Z 2022-01-12T14:36:27.793407Z 2022-01-12T14:36:27.751255Z 2022-01-12T14:36:27.699034Z 2022-01-12T14:36:27.657503Z 2022-01-12T14:36:27.614295Z 2022-01-12T14:36:27.569330Z 2022-01-12T14:36:27.514301Z 2022-01-12T14:36:27.447984Z 2022-01-12T14:36:27.358380Z 2022-01-12T14:36:27.215946Z 2022-01-12T14:36:27.058579Z 2022-01-12T14:36:26.885515Z 2022-01-12T14:36:26.754382Z 2022-01-12T14:36:26.658206Z 2022-01-12T14:36:26.561137Z 2022-01-12T14:36:26.476138Z 2022-01-12T14:36:26.373386Z 2022-01-12T14:36:26.256974Z 2022-01-12T14:36:26.140714Z 2022-01-12T14:36:26.055222Z 2022-01-12T14:36:25.949546Z 2022-01-12T14:36:25.738099Z 2022-01-12T14:36:25.836989Z 2022-01-12T14:36:25.611218Z 2022-01-12T14:36:25.491509Z 2022-01-12T14:36:25.358325Z 2022-01-12T14:36:25.228717Z 2022-01-12T14:36:25.090797Z 2022-01-12T14:36:25.034075Z 2022-01-12T14:36:24.967026Z 2022-01-12T14:36:24.909951Z 2022-01-12T14:36:24.854328Z 2022-01-12T14:36:24.793082Z 2022-01-12T14:36:24.733996Z 2022-01-12T14:36:24.674971Z 2022-01-12T14:36:24.617077Z 2022-01-12T14:36:24.560171Z 2022-01-12T14:36:24.499092Z 2022-01-12T14:36:24.422429Z 2022-01-12T14:36:24.342945Z 2022-01-12T14:36:24.274224Z 2022-01-12T14:36:24.184879Z 2022-01-12T14:36:24.112032Z 2022-01-12T14:36:23.902162Z 2022-01-12T14:36:24.064414Z 2022-01-12T14:36:23.740349Z 2022-01-12T14:36:23.658883Z 2022-01-12T14:36:23.612147Z 2022-01-12T14:36:23.552347Z 2022-01-12T14:36:23.461244Z 2022-01-12T14:36:23.275106Z 2022-01-12T14:36:23.375972Z 2022-01-12T14:36:23.160084Z 2022-01-12T14:36:23.075668Z 2022-01-12T14:36:22.970229Z 2022-01-12T14:36:22.866270Z 2022-01-12T14:36:22.775420Z 2022-01-12T14:36:12.159052Z 2022-01-12T14:36:11.670264Z 2022-01-12T14:36:11.271704Z 2022-01-12T14:36:11.016496Z 2022-01-12T14:36:10.713219Z 2022-01-12T14:36:10.569869Z 2022-01-12T14:36:10.333328Z 2022-01-12T14:36:10.178617Z 2022-01-12T14:36:10.018442Z 2022-01-12T14:36:09.749225Z 2022-01-12T14:36:09.483896Z 2022-01-12T14:36:09.176466Z 2022-01-12T14:36:08.993115Z 2022-01-12T14:36:08.722184Z 2022-01-12T14:36:08.611654Z 2022-01-12T14:36:08.449539Z 2022-01-12T14:36:05.387631Z 2022-01-12T14:36:05.281255Z 2022-01-12T14:36:05.219796Z 2022-01-12T14:36:05.134607Z 2022-01-12T14:36:05.056334Z 2022-01-12T14:41:44.450720Z 2022-01-12T14:41:44.327833Z 2022-01-12T14:41:44.245989Z 2022-01-12T14:41:44.174684Z 2022-01-12T14:41:44.101087Z 2022-01-12T14:41:44.034255Z 2022-01-12T14:41:43.939237Z 2022-01-12T14:41:43.829212Z 2022-01-12T14:41:43.740080Z 2022-01-12T14:41:43.654062Z 2022-01-12T14:41:43.544129Z 2022-01-12T14:41:43.423823Z 2022-01-12T14:41:43.290454Z 2022-01-12T14:41:43.149794Z 2022-01-12T14:41:43.024261Z 2022-01-12T14:41:42.890551Z 2022-01-12T14:41:42.769215Z 2022-01-12T14:41:42.659150Z 2022-01-12T14:41:42.509055Z 2022-01-12T14:41:42.394137Z 2022-01-12T14:41:42.315108Z 2022-01-12T14:41:42.196990Z 2022-01-12T14:41:42.079023Z 2022-01-12T14:41:41.969146Z 2022-01-12T14:41:41.870259Z 2022-01-12T14:41:41.744319Z 2022-01-12T14:41:41.629144Z 2022-01-12T14:41:41.504285Z 2022-01-12T14:41:41.365585Z 2022-01-12T14:41:41.212182Z 2022-01-12T14:41:41.138168Z 2022-01-12T14:41:41.068951Z 2022-01-12T14:41:40.989575Z 2022-01-12T14:41:40.896658Z 2022-01-12T14:41:40.761427Z 2022-01-12T14:41:40.829023Z 2022-01-12T14:41:40.684054Z 2022-01-12T14:41:40.631110Z 2022-01-12T14:41:40.580133Z 2022-01-12T14:41:40.517991Z 2022-01-12T14:41:40.450119Z 2022-01-12T14:41:17.253357Z 2022-01-12T14:41:17.209899Z 2022-01-12T14:41:40.365110Z 2022-01-12T14:41:40.276181Z 2022-01-12T14:41:40.161260Z 2022-01-12T14:41:17.132979Z 2022-01-12T14:41:17.072191Z 2022-01-12T14:41:40.027163Z 2022-01-12T14:41:17.032776Z 2022-01-12T14:41:16.990132Z 2022-01-12T14:41:16.922100Z 2022-01-12T14:41:16.864254Z 2022-01-12T14:41:16.822063Z 2022-01-12T14:41:16.782080Z 2022-01-12T14:41:16.745455Z 2022-01-12T14:41:16.704971Z 2022-01-12T14:41:16.665266Z 2022-01-12T14:41:16.622878Z 2022-01-12T14:41:16.583398Z 2022-01-12T14:41:16.522991Z 2022-01-12T14:41:16.357228Z 2022-01-12T14:41:16.254194Z 2022-01-12T14:41:16.180109Z 2022-01-12T14:41:16.095655Z 2022-01-12T14:41:16.030176Z 2022-01-12T14:41:15.883033Z 2022-01-12T14:41:15.798170Z 2022-01-12T14:41:15.614819Z 2022-01-12T14:41:15.428003Z 2022-01-12T14:41:15.355805Z 2022-01-12T14:41:15.250396Z 2022-01-12T14:41:15.201187Z 2022-01-12T14:41:15.067078Z 2022-01-12T14:41:14.977800Z 2022-01-12T14:41:14.811719Z 2022-01-12T14:41:14.672112Z 2022-01-12T14:41:14.475284Z 2022-01-12T14:41:14.401290Z 2022-01-12T14:41:14.295230Z 2022-01-12T14:41:14.151965Z 2022-01-12T14:41:14.078174Z 2022-01-12T14:41:13.934273Z 2022-01-12T14:41:13.784696Z 2022-01-12T14:41:13.716837Z 2022-01-12T14:41:13.661629Z 2022-01-12T14:41:13.469197Z 2022-01-12T14:41:13.169879Z 2022-01-12T14:41:12.802264Z 2022-01-12T14:41:12.591025Z 2022-01-12T14:41:03.304096Z 2022-01-12T14:41:03.148329Z 2022-01-12T14:41:02.973443Z 2022-01-12T14:41:02.794227Z 2022-01-12T14:41:02.689245Z 2022-01-12T14:41:02.593577Z 2022-01-12T14:41:02.502370Z 2022-01-12T14:41:02.070530Z 2022-01-12T14:41:00.734657Z 2022-01-12T14:40:59.747678Z", + "modified_diff": 50.3378704, + "started": "2022-01-12T14:14:10.935621Z 2022-01-12T14:14:12.409019Z 2022-01-12T14:14:11.509150Z 2022-01-12T14:14:10.492300Z 2022-01-12T14:14:10.217759Z 2022-01-12T14:14:11.302198Z 2022-01-12T14:14:10.018261Z 2022-01-12T14:14:11.068906Z 2022-01-12T14:14:09.792241Z 2022-01-12T14:14:10.394367Z 2022-01-12T14:14:09.610047Z 2022-01-12T14:14:07.120145Z 2022-01-12T14:14:05.970448Z 2022-01-12T14:14:05.794924Z 2022-01-12T14:14:05.579080Z 2022-01-12T14:14:05.870499Z 2022-01-12T14:14:04.889263Z 2022-01-12T14:14:04.928077Z 2022-01-12T14:14:04.142272Z 2022-01-12T14:14:03.798416Z 2022-01-12T14:14:03.451825Z 2022-01-12T14:14:03.337768Z 2022-01-12T14:14:03.003203Z 2022-01-12T14:14:02.682012Z 2022-01-12T14:14:02.528044Z 2022-01-12T14:14:02.175858Z 2022-01-12T14:14:02.050945Z 2022-01-12T14:13:57.936631Z 2022-01-12T14:13:57.681986Z 2022-01-12T14:13:57.441100Z 2022-01-12T14:13:57.294329Z 2022-01-12T14:13:57.107745Z 2022-01-12T14:13:56.878092Z 2022-01-12T14:13:56.002238Z 2022-01-12T14:13:55.867484Z 2022-01-12T14:13:55.787507Z 2022-01-12T14:14:57.585116Z 2022-01-12T14:14:57.276814Z 2022-01-12T14:14:56.943938Z 2022-01-12T14:14:57.208879Z 2022-01-12T14:14:56.880273Z 2022-01-12T14:14:56.403159Z 2022-01-12T14:14:56.316036Z 2022-01-12T14:14:55.607533Z 2022-01-12T14:14:55.280054Z 2022-01-12T14:14:55.003184Z 2022-01-12T14:14:49.389465Z 2022-01-12T14:14:48.773819Z 2022-01-12T14:14:48.268836Z 2022-01-12T14:14:54.912320Z 2022-01-12T14:14:47.377941Z 2022-01-12T14:14:46.754657Z 2022-01-12T14:14:46.256553Z 2022-01-12T14:14:46.127903Z 2022-01-12T14:14:45.931549Z 2022-01-12T14:14:45.080065Z 2022-01-12T14:14:44.678892Z 2022-01-12T14:14:44.052324Z 2022-01-12T14:14:43.699551Z 2022-01-12T14:14:43.334900Z 2022-01-12T14:14:42.734096Z 2022-01-12T14:14:42.513473Z 2022-01-12T14:14:41.938352Z 2022-01-12T14:14:41.847844Z 2022-01-12T14:14:41.654289Z 2022-01-12T14:14:41.205674Z 2022-01-12T14:14:40.599535Z 2022-01-12T14:14:40.449036Z 2022-01-12T14:14:39.708935Z 2022-01-12T14:14:39.662455Z 2022-01-12T14:14:39.153632Z 2022-01-12T14:14:38.887149Z 2022-01-12T14:14:38.413163Z 2022-01-12T14:14:24.531190Z 2022-01-12T14:14:18.133020Z 2022-01-12T14:14:24.715824Z 2022-01-12T14:14:17.284994Z 2022-01-12T14:14:22.565185Z 2022-01-12T14:14:16.761815Z 2022-01-12T14:14:22.136912Z 2022-01-12T14:14:16.242410Z 2022-01-12T14:14:22.379498Z 2022-01-12T14:14:15.712397Z 2022-01-12T14:14:20.295918Z 2022-01-12T14:14:15.127133Z 2022-01-12T14:14:17.044119Z 2022-01-12T14:14:14.219484Z 2022-01-12T14:14:15.590598Z 2022-01-12T14:14:13.565058Z 2022-01-12T14:14:15.994998Z 2022-01-12T14:14:13.274414Z 2022-01-12T14:14:15.608228Z 2022-01-12T14:14:12.865834Z 2022-01-12T14:14:15.624048Z 2022-01-12T14:14:12.434055Z 2022-01-12T14:14:14.228040Z 2022-01-12T14:14:11.880489Z 2022-01-12T14:14:13.243868Z 2022-01-12T14:14:11.343071Z 2022-01-12T14:14:12.894637Z 2022-01-12T14:28:06.245698Z 2022-01-12T14:28:05.849096Z 2022-01-12T14:28:05.309998Z 2022-01-12T14:28:04.781121Z 2022-01-12T14:28:04.370531Z 2022-01-12T14:28:04.113997Z 2022-01-12T14:28:03.621034Z 2022-01-12T14:28:03.066944Z 2022-01-12T14:28:02.897605Z 2022-01-12T14:28:02.095006Z 2022-01-12T14:28:02.541011Z 2022-01-12T14:28:01.568603Z 2022-01-12T14:28:01.007905Z 2022-01-12T14:28:00.593149Z 2022-01-12T14:28:00.214042Z 2022-01-12T14:28:00.072032Z 2022-01-12T14:28:00.139551Z 2022-01-12T14:27:59.473654Z 2022-01-12T14:27:59.120293Z 2022-01-12T14:27:58.590871Z 2022-01-12T14:27:58.532247Z 2022-01-12T14:27:52.354265Z 2022-01-12T14:27:58.142189Z 2022-01-12T14:27:51.990167Z 2022-01-12T14:27:51.250899Z 2022-01-12T14:27:50.863815Z 2022-01-12T14:27:51.256778Z 2022-01-12T14:27:50.364648Z 2022-01-12T14:27:50.243427Z 2022-01-12T14:27:48.535619Z 2022-01-12T14:27:48.133465Z 2022-01-12T14:27:58.097050Z 2022-01-12T14:27:47.703833Z 2022-01-12T14:27:47.197414Z 2022-01-12T14:27:57.798332Z 2022-01-12T14:27:46.628361Z 2022-01-12T14:27:46.382193Z 2022-01-12T14:27:45.534260Z 2022-01-12T14:27:45.729820Z 2022-01-12T14:27:44.684910Z 2022-01-12T14:27:44.472648Z 2022-01-12T14:27:43.722631Z 2022-01-12T14:27:43.160911Z 2022-01-12T14:27:42.430880Z 2022-01-12T14:27:42.280349Z 2022-01-12T14:27:41.580363Z 2022-01-12T14:27:40.977614Z 2022-01-12T14:27:40.585812Z 2022-01-12T14:27:39.928058Z 2022-01-12T14:27:39.694020Z 2022-01-12T14:27:39.733852Z 2022-01-12T14:27:39.332347Z 2022-01-12T14:27:38.757615Z 2022-01-12T14:27:38.335203Z 2022-01-12T14:27:38.398952Z 2022-01-12T14:27:37.889126Z 2022-01-12T14:27:37.738200Z 2022-01-12T14:27:37.017217Z 2022-01-12T14:27:36.723837Z 2022-01-12T14:27:36.352407Z 2022-01-12T14:27:35.803622Z 2022-01-12T14:27:35.323255Z 2022-01-12T14:27:34.877185Z 2022-01-12T14:27:34.029378Z 2022-01-12T14:27:33.517077Z 2022-01-12T14:27:33.157173Z 2022-01-12T14:27:32.992973Z 2022-01-12T14:27:32.777103Z 2022-01-12T14:27:32.096893Z 2022-01-12T14:27:31.583162Z 2022-01-12T14:27:31.414481Z 2022-01-12T14:27:31.351140Z 2022-01-12T14:27:31.237540Z 2022-01-12T14:27:31.141865Z 2022-01-12T14:27:30.901623Z 2022-01-12T14:27:19.970262Z 2022-01-12T14:27:18.683442Z 2022-01-12T14:27:18.469606Z 2022-01-12T14:27:16.702973Z 2022-01-12T14:27:17.382080Z 2022-01-12T14:27:16.209457Z 2022-01-12T14:27:15.375031Z 2022-01-12T14:27:14.745483Z 2022-01-12T14:27:14.433353Z 2022-01-12T14:27:13.390933Z 2022-01-12T14:27:13.686289Z 2022-01-12T14:27:12.187713Z 2022-01-12T14:27:11.212253Z 2022-01-12T14:27:10.796042Z 2022-01-12T14:27:04.755527Z 2022-01-12T14:27:04.418505Z 2022-01-12T14:27:04.194236Z 2022-01-12T14:27:03.721344Z 2022-01-12T14:27:03.626385Z 2022-01-12T14:27:03.271960Z 2022-01-12T14:27:02.173801Z 2022-01-12T14:27:01.925425Z 2022-01-12T14:27:01.729884Z 2022-01-12T14:27:00.939442Z 2022-01-12T14:27:00.861653Z 2022-01-12T14:32:27.818243Z 2022-01-12T14:32:27.466474Z 2022-01-12T14:32:27.795957Z 2022-01-12T14:32:26.299061Z 2022-01-12T14:32:26.476090Z 2022-01-12T14:32:25.778765Z 2022-01-12T14:32:25.813697Z 2022-01-12T14:32:25.001668Z 2022-01-12T14:32:25.114364Z 2022-01-12T14:32:24.561911Z 2022-01-12T14:32:24.401729Z 2022-01-12T14:32:23.612061Z 2022-01-12T14:32:23.058887Z 2022-01-12T14:32:22.869578Z 2022-01-12T14:32:23.072721Z 2022-01-12T14:32:22.253311Z 2022-01-12T14:32:22.280075Z 2022-01-12T14:32:21.602994Z 2022-01-12T14:32:21.423228Z 2022-01-12T14:32:20.284680Z 2022-01-12T14:32:19.997509Z 2022-01-12T14:32:19.121569Z 2022-01-12T14:32:18.617442Z 2022-01-12T14:32:18.126637Z 2022-01-12T14:32:17.572974Z 2022-01-12T14:32:17.576071Z 2022-01-12T14:32:17.291124Z 2022-01-12T14:32:16.278969Z 2022-01-12T14:32:16.197505Z 2022-01-12T14:32:15.320834Z 2022-01-12T14:32:15.039159Z 2022-01-12T14:32:14.571330Z 2022-01-12T14:32:13.844059Z 2022-01-12T14:32:14.395001Z 2022-01-12T14:32:13.377177Z 2022-01-12T14:32:12.645924Z 2022-01-12T14:32:11.877050Z 2022-01-12T14:32:12.149263Z 2022-01-12T14:32:11.078277Z 2022-01-12T14:32:10.403835Z 2022-01-12T14:32:10.150967Z 2022-01-12T14:32:09.623147Z 2022-01-12T14:32:09.198994Z 2022-01-12T14:32:09.032936Z 2022-01-12T14:32:08.315048Z 2022-01-12T14:32:08.175115Z 2022-01-12T14:32:08.127307Z 2022-01-12T14:32:07.555686Z 2022-01-12T14:32:07.353168Z 2022-01-12T14:32:06.663670Z 2022-01-12T14:32:06.220918Z 2022-01-12T14:32:05.961240Z 2022-01-12T14:32:05.273124Z 2022-01-12T14:32:05.156454Z 2022-01-12T14:32:04.342247Z 2022-01-12T14:32:04.175249Z 2022-01-12T14:32:03.885474Z 2022-01-12T14:32:03.473159Z 2022-01-12T14:32:03.383241Z 2022-01-12T14:32:02.872502Z 2022-01-12T14:32:02.239405Z 2022-01-12T14:32:01.727555Z 2022-01-12T14:32:01.453299Z 2022-01-12T14:32:00.856417Z 2022-01-12T14:32:00.803907Z 2022-01-12T14:32:00.527747Z 2022-01-12T14:32:00.168399Z 2022-01-12T14:31:59.882121Z 2022-01-12T14:31:59.424287Z 2022-01-12T14:31:59.356839Z 2022-01-12T14:31:58.960005Z 2022-01-12T14:31:50.559742Z 2022-01-12T14:31:58.761019Z 2022-01-12T14:31:58.647686Z 2022-01-12T14:31:49.782941Z 2022-01-12T14:31:50.224615Z 2022-01-12T14:31:49.022387Z 2022-01-12T14:31:49.047803Z 2022-01-12T14:31:48.542855Z 2022-01-12T14:31:48.301063Z 2022-01-12T14:31:47.953274Z 2022-01-12T14:31:48.066288Z 2022-01-12T14:31:47.329007Z 2022-01-12T14:31:47.436011Z 2022-01-12T14:31:46.780779Z 2022-01-12T14:31:46.739193Z 2022-01-12T14:31:45.982107Z 2022-01-12T14:31:45.624809Z 2022-01-12T14:31:44.655494Z 2022-01-12T14:31:44.542087Z 2022-01-12T14:31:43.020941Z 2022-01-12T14:31:42.646274Z 2022-01-12T14:31:41.474313Z 2022-01-12T14:31:41.179628Z 2022-01-12T14:31:32.921879Z 2022-01-12T14:31:32.323978Z 2022-01-12T14:31:31.849993Z 2022-01-12T14:31:30.472052Z 2022-01-12T14:31:30.313548Z 2022-01-12T14:31:30.176213Z 2022-01-12T14:37:08.848464Z 2022-01-12T14:37:04.238657Z 2022-01-12T14:37:03.605593Z 2022-01-12T14:37:03.373800Z 2022-01-12T14:37:02.502654Z 2022-01-12T14:37:02.180116Z 2022-01-12T14:37:01.868525Z 2022-01-12T14:37:01.367102Z 2022-01-12T14:37:00.879069Z 2022-01-12T14:37:00.547331Z 2022-01-12T14:36:59.829194Z 2022-01-12T14:36:59.001936Z 2022-01-12T14:36:58.874176Z 2022-01-12T14:36:58.519584Z 2022-01-12T14:36:58.127829Z 2022-01-12T14:36:57.901551Z 2022-01-12T14:36:57.156242Z 2022-01-12T14:36:56.673001Z 2022-01-12T14:36:56.335961Z 2022-01-12T14:36:55.678649Z 2022-01-12T14:36:55.132703Z 2022-01-12T14:36:55.281406Z 2022-01-12T14:36:54.270793Z 2022-01-12T14:36:54.291843Z 2022-01-12T14:36:53.108102Z 2022-01-12T14:36:52.978906Z 2022-01-12T14:36:52.203518Z 2022-01-12T14:36:51.765101Z 2022-01-12T14:36:51.056912Z 2022-01-12T14:36:50.614121Z 2022-01-12T14:36:50.080816Z 2022-01-12T14:36:49.655425Z 2022-01-12T14:36:48.862706Z 2022-01-12T14:36:48.703961Z 2022-01-12T14:36:47.608903Z 2022-01-12T14:36:47.492355Z 2022-01-12T14:36:46.868924Z 2022-01-12T14:36:46.605447Z 2022-01-12T14:36:46.050945Z 2022-01-12T14:36:45.983038Z 2022-01-12T14:36:45.107707Z 2022-01-12T14:36:44.800973Z 2022-01-12T14:36:43.841680Z 2022-01-12T14:36:43.158932Z 2022-01-12T14:36:43.940840Z 2022-01-12T14:36:43.280860Z 2022-01-12T14:36:42.450018Z 2022-01-12T14:36:42.261036Z 2022-01-12T14:36:41.659987Z 2022-01-12T14:36:41.446799Z 2022-01-12T14:36:41.081963Z 2022-01-12T14:36:41.115346Z 2022-01-12T14:36:40.008231Z 2022-01-12T14:36:39.480844Z 2022-01-12T14:36:38.672443Z 2022-01-12T14:36:38.044706Z 2022-01-12T14:36:37.489941Z 2022-01-12T14:36:37.452731Z 2022-01-12T14:36:36.900796Z 2022-01-12T14:36:38.331063Z 2022-01-12T14:36:36.177086Z 2022-01-12T14:36:36.066460Z 2022-01-12T14:36:35.669572Z 2022-01-12T14:36:35.380941Z 2022-01-12T14:36:35.529252Z 2022-01-12T14:36:35.162965Z 2022-01-12T14:36:34.487515Z 2022-01-12T14:36:33.486105Z 2022-01-12T14:36:32.885290Z 2022-01-12T14:36:32.110408Z 2022-01-12T14:36:31.767939Z 2022-01-12T14:36:31.899375Z 2022-01-12T14:36:31.133954Z 2022-01-12T14:36:31.298043Z 2022-01-12T14:36:30.845058Z 2022-01-12T14:36:30.546637Z 2022-01-12T14:36:30.308935Z 2022-01-12T14:36:30.031502Z 2022-01-12T14:36:29.785069Z 2022-01-12T14:36:19.500075Z 2022-01-12T14:36:19.309314Z 2022-01-12T14:36:18.823789Z 2022-01-12T14:36:18.724051Z 2022-01-12T14:36:18.442129Z 2022-01-12T14:36:17.688490Z 2022-01-12T14:36:17.421228Z 2022-01-12T14:36:16.878099Z 2022-01-12T14:36:16.974097Z 2022-01-12T14:36:16.091283Z 2022-01-12T14:36:16.100668Z 2022-01-12T14:36:15.296018Z 2022-01-12T14:36:14.913088Z 2022-01-12T14:36:14.437807Z 2022-01-12T14:36:14.427600Z 2022-01-12T14:36:14.232992Z 2022-01-12T14:36:07.142409Z 2022-01-12T14:36:06.561114Z 2022-01-12T14:36:06.347450Z 2022-01-12T14:36:05.984352Z 2022-01-12T14:36:05.893662Z 2022-01-12T14:42:03.515566Z 2022-01-12T14:42:04.372795Z 2022-01-12T14:42:03.567063Z 2022-01-12T14:42:02.755061Z 2022-01-12T14:42:02.568481Z 2022-01-12T14:42:01.728930Z 2022-01-12T14:42:01.455383Z 2022-01-12T14:42:01.222790Z 2022-01-12T14:42:00.930321Z 2022-01-12T14:42:00.511917Z 2022-01-12T14:42:00.254479Z 2022-01-12T14:41:59.565104Z 2022-01-12T14:41:59.598561Z 2022-01-12T14:41:58.736229Z 2022-01-12T14:41:58.303928Z 2022-01-12T14:41:58.192039Z 2022-01-12T14:41:57.559448Z 2022-01-12T14:41:56.958574Z 2022-01-12T14:41:56.929065Z 2022-01-12T14:41:56.512198Z 2022-01-12T14:41:55.775190Z 2022-01-12T14:41:55.371253Z 2022-01-12T14:41:54.662842Z 2022-01-12T14:41:53.972912Z 2022-01-12T14:41:53.495264Z 2022-01-12T14:41:53.391194Z 2022-01-12T14:41:52.956172Z 2022-01-12T14:41:51.600906Z 2022-01-12T14:41:51.242552Z 2022-01-12T14:41:50.951375Z 2022-01-12T14:41:49.514036Z 2022-01-12T14:41:49.589940Z 2022-01-12T14:41:49.304701Z 2022-01-12T14:41:48.896679Z 2022-01-12T14:41:47.791810Z 2022-01-12T14:41:48.592350Z 2022-01-12T14:41:47.502304Z 2022-01-12T14:41:47.457996Z 2022-01-12T14:41:46.900070Z 2022-01-12T14:41:46.913802Z 2022-01-12T14:41:46.560222Z 2022-01-12T14:41:37.047998Z 2022-01-12T14:41:37.091236Z 2022-01-12T14:41:46.363845Z 2022-01-12T14:41:45.790101Z 2022-01-12T14:41:45.608412Z 2022-01-12T14:41:35.353867Z 2022-01-12T14:41:34.818948Z 2022-01-12T14:41:45.165167Z 2022-01-12T14:41:34.501643Z 2022-01-12T14:41:33.596068Z 2022-01-12T14:41:33.176840Z 2022-01-12T14:41:32.543366Z 2022-01-12T14:41:32.337343Z 2022-01-12T14:41:31.751892Z 2022-01-12T14:41:31.413117Z 2022-01-12T14:41:30.954302Z 2022-01-12T14:41:30.189928Z 2022-01-12T14:41:28.984712Z 2022-01-12T14:41:28.658202Z 2022-01-12T14:41:27.842334Z 2022-01-12T14:41:27.398476Z 2022-01-12T14:41:26.978462Z 2022-01-12T14:41:26.643076Z 2022-01-12T14:41:25.566810Z 2022-01-12T14:41:25.609931Z 2022-01-12T14:41:24.751065Z 2022-01-12T14:41:24.417457Z 2022-01-12T14:41:23.738831Z 2022-01-12T14:41:23.205056Z 2022-01-12T14:41:22.711653Z 2022-01-12T14:41:22.731265Z 2022-01-12T14:41:22.157848Z 2022-01-12T14:41:22.000500Z 2022-01-12T14:41:21.767189Z 2022-01-12T14:41:21.413055Z 2022-01-12T14:41:20.855821Z 2022-01-12T14:41:20.574227Z 2022-01-12T14:41:19.910058Z 2022-01-12T14:41:20.537743Z 2022-01-12T14:41:19.313593Z 2022-01-12T14:41:20.008236Z 2022-01-12T14:41:18.755988Z 2022-01-12T14:41:18.864831Z 2022-01-12T14:41:18.329088Z 2022-01-12T14:41:18.655710Z 2022-01-12T14:41:18.182896Z 2022-01-12T14:41:18.460210Z 2022-01-12T14:41:17.816643Z 2022-01-12T14:41:18.500164Z 2022-01-12T14:41:07.193864Z 2022-01-12T14:41:07.053532Z 2022-01-12T14:41:06.551702Z 2022-01-12T14:41:06.206288Z 2022-01-12T14:41:05.003075Z 2022-01-12T14:41:05.863997Z 2022-01-12T14:41:04.336219Z 2022-01-12T14:41:04.604606Z 2022-01-12T14:41:01.410837Z 2022-01-12T14:40:59.982431Z", + "started_diff": 62.43377, + "successful_job_ids": " [99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 88, 86, 85, 84, 83, 82, 81, 79, 78, 77, 76, 75, 74, 73, 72, 71, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 172, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 154, 153, 152, 151, 150, 149, 147, 146, 145, 144, 143, 142, 141, 138, 137, 136, 135, 134, 132, 131, 129, 128, 126, 124, 123, 122, 121, 120, 119, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 472, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 572, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473]" + }, + "started": "2022-01-12T15:10:16.608189+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-api.json new file mode 100644 index 00000000..af943fb1 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-api.json @@ -0,0 +1,1244 @@ +{ + "ended": "2022-01-17T12:11:06.977922+00:00", + "id": "run-2022-01-17T09_07_03_881_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2257.5, + "max": 25632.533333333333, + "mean": 5341.128787878788, + "median": 3347.233333333333, + "min": 1504.4666666666667, + "non_zero_mean": 5341.128787878788, + "non_zero_median": 3347.233333333333, + "percentile25": 2786.366666666667, + "percentile75": 5043.866666666667, + "percentile90": 12888.593333333338, + "percentile99": 23508.880000000005, + "percentile999": 25519.89920000001, + "range": 24128.066666666666, + "samples": 264, + "stdev": 4878.21185792271, + "sum": 1410057.9999999995 + } + }, + "cpu": { + "system": { + "iqr": 0.18416666666667064, + "max": 1.829333333333322, + "mean": 0.27597222222222223, + "median": 0.11233333333333158, + "min": 0.03333333333333333, + "non_zero_mean": 0.27597222222222223, + "non_zero_median": 0.11233333333333158, + "percentile25": 0.06266666666666651, + "percentile75": 0.24683333333333715, + "percentile90": 0.8780666666666672, + "percentile99": 1.7520933333333335, + "percentile999": 1.8142546666666612, + "range": 1.7959999999999885, + "samples": 264, + "stdev": 0.4024502531227407, + "sum": 72.85666666666665 + }, + "user": { + "iqr": 0.5374999999999848, + "max": 6.284000000000014, + "mean": 1.3725227272727274, + "median": 0.7806666666666692, + "min": 0.41599999999998544, + "non_zero_mean": 1.3725227272727274, + "non_zero_median": 0.7806666666666692, + "percentile25": 0.598000000000017, + "percentile75": 1.1355000000000017, + "percentile90": 3.9139333333333095, + "percentile99": 6.200433333333354, + "percentile999": 6.26944733333335, + "range": 5.868000000000029, + "samples": 264, + "stdev": 1.4816152404209197, + "sum": 362.34600000000006 + } + }, + "entropy_available_bits": { + "iqr": 1.9333333333333331, + "max": 381.66666666666663, + "mean": 20.10328282828283, + "median": 1.6666666666666665, + "min": 0.2, + "non_zero_mean": 20.10328282828283, + "non_zero_median": 1.6666666666666665, + "percentile25": 0.7333333333333333, + "percentile75": 2.6666666666666665, + "percentile90": 4.713333333333335, + "percentile99": 370.84866666666665, + "percentile999": 380.12373333333346, + "range": 381.46666666666664, + "samples": 264, + "stdev": 68.57742331671251, + "sum": 5307.266666666666 + }, + "memory": { + "used": { + "iqr": 222306304.0, + "max": 15878778880.0, + "mean": 6508684225.939394, + "median": 5936230400.0, + "min": 5685444608.0, + "non_zero_mean": 6508684225.939394, + "non_zero_median": 5936230400.0, + "percentile25": 5873268736.0, + "percentile75": 6095575040.0, + "percentile90": 7351370137.600002, + "percentile99": 15299473981.440002, + "percentile999": 15867363282.944, + "range": 10193334272.0, + "samples": 264, + "stdev": 1838634701.6153216, + "sum": 1718292635648.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 17.53333333333333, + "mean": 0.16237373737373736, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.2974358974358973, + "non_zero_median": 0.8, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.9846666666666732, + "percentile999": 15.376733333333522, + "range": 17.53333333333333, + "samples": 264, + "stdev": 1.2898151965629951, + "sum": 42.866666666666674 + } + }, + "writes_completed": { + "total": { + "iqr": 83.38333333333333, + "max": 1486.4666666666665, + "mean": 121.3229797979798, + "median": 29.966666666666665, + "min": 0.0, + "non_zero_mean": 125.60496732026144, + "non_zero_median": 31.466666666666665, + "percentile25": 1.4666666666666666, + "percentile75": 84.85, + "percentile90": 349.87333333333356, + "percentile99": 1333.8500000000001, + "percentile999": 1460.1842000000022, + "range": 1486.4666666666665, + "samples": 264, + "stdev": 264.96639193930093, + "sum": 32029.266666666677 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 938.5666666666666, + "max": 3124.2, + "mean": 1623.457201420024, + "median": 1487.3666666666668, + "min": 541.2, + "non_zero_mean": 1623.457201420024, + "non_zero_median": 1487.3666666666668, + "percentile25": 1150.5333333333333, + "percentile75": 2089.1, + "percentile90": 2424.2000000000003, + "percentile99": 3033.882, + "percentile999": 3111.8565333333345, + "range": 2583.0, + "samples": 264, + "stdev": 593.4054252235441, + "sum": 428592.70117488614 + } + }, + "cpu": { + "system": { + "iqr": 0.044833333333332816, + "max": 0.25400000000000017, + "mean": 0.08455049932048135, + "median": 0.07133333333333478, + "min": 0.012000000000000456, + "non_zero_mean": 0.08455049932048135, + "non_zero_median": 0.07133333333333478, + "percentile25": 0.052666666666668034, + "percentile75": 0.09750000000000085, + "percentile90": 0.15866666666666768, + "percentile99": 0.22604666666666595, + "percentile999": 0.25382466666666653, + "range": 0.24199999999999972, + "samples": 264, + "stdev": 0.04529169809850803, + "sum": 22.3213318206071 + }, + "user": { + "iqr": 0.1195000000000003, + "max": 0.8879999999999996, + "mean": 0.20050004988982006, + "median": 0.19733333333333292, + "min": 0.01933333333333091, + "non_zero_mean": 0.20050004988982006, + "non_zero_median": 0.19733333333333292, + "percentile25": 0.13266666666666635, + "percentile75": 0.25216666666666665, + "percentile90": 0.3067333333333348, + "percentile99": 0.5392133333333328, + "percentile999": 0.8322440000000043, + "range": 0.8686666666666687, + "samples": 264, + "stdev": 0.10567234539630528, + "sum": 52.93201317091248 + } + }, + "entropy_available_bits": { + "iqr": 1.75, + "max": 214.6, + "mean": 11.234089616368966, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 22.99069502884812, + "non_zero_median": 1.8, + "percentile25": 0.0, + "percentile75": 1.75, + "percentile90": 2.9133333333333344, + "percentile99": 212.99733333333336, + "percentile999": 214.38960000000003, + "range": 214.6, + "samples": 264, + "stdev": 45.78125411280827, + "sum": 2965.799658721406 + }, + "memory": { + "used": { + "iqr": 23364608.0, + "max": 1041866752.0, + "mean": 430433186.90909094, + "median": 389378048.0, + "min": 367730688.0, + "non_zero_mean": 430433186.90909094, + "non_zero_median": 389378048.0, + "percentile25": 381287424.0, + "percentile75": 404652032.0, + "percentile90": 506468352.0000008, + "percentile99": 1005602693.1200001, + "percentile999": 1041545732.0960001, + "range": 674136064.0, + "samples": 264, + "stdev": 126945251.15940316, + "sum": 113634361344.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 6007.466666666666, + "max": 59702808.46666667, + "mean": 1564005.3679292928, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 4256674.403436426, + "non_zero_median": 12561.066666666668, + "percentile25": 0.0, + "percentile75": 6007.466666666666, + "percentile90": 40632.3200000002, + "percentile99": 58472222.61066668, + "percentile999": 59640040.81653334, + "range": 59702808.46666667, + "samples": 264, + "stdev": 9426015.276319362, + "sum": 412897417.1333331 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 430.243902, + "mean": 6.6558471818181815, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 46.24062252631579, + "non_zero_median": 0.049813, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.013120200000000035, + "percentile99": 302.44934569000026, + "percentile999": 422.35425179000066, + "range": 430.243902, + "samples": 264, + "stdev": 46.636280486436235, + "sum": 1757.1436560000002 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.29813275, + "max": 415.244339, + "mean": 21.719943950757575, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 53.58939442056075, + "non_zero_median": 0.716868, + "percentile25": 0.0, + "percentile75": 0.29813275, + "percentile90": 56.88732260000004, + "percentile99": 360.79569035000003, + "percentile999": 407.3560056310007, + "range": 415.244339, + "samples": 264, + "stdev": 69.37944420002361, + "sum": 5734.065203000002 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 4.466666666666667, + "mean": 0.17853535353535355, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.3666666666666667, + "non_zero_median": 3.533333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.140666666666667, + "percentile999": 4.449133333333335, + "range": 4.466666666666667, + "samples": 264, + "stdev": 0.7754103545931453, + "sum": 47.13333333333333 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9996.466666666667, + "mean": 464.139898989899, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8752.352380952381, + "non_zero_median": 9699.433333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9979.546666666667, + "percentile999": 9995.256866666667, + "range": 9996.466666666667, + "samples": 264, + "stdev": 1992.8898876653784, + "sum": 122532.93333333335 + }, + "pg_stat_database_blks_hit": { + "iqr": 22323.550000000003, + "max": 210551.33333333334, + "mean": 25703.098737373737, + "median": 26219.86666666667, + "min": 2496.733333333333, + "non_zero_mean": 25703.098737373737, + "non_zero_median": 26219.86666666667, + "percentile25": 11018.716666666667, + "percentile75": 33342.26666666667, + "percentile90": 41052.42666666667, + "percentile99": 67128.67000000016, + "percentile999": 199151.12493333436, + "range": 208054.6, + "samples": 264, + "stdev": 20471.47320997273, + "sum": 6785618.0666666655 + }, + "pg_stat_database_blks_read": { + "iqr": 0.6166666666666667, + "max": 213.86666666666667, + "mean": 1.4196969696969697, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7679245283018867, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6833333333333333, + "percentile90": 2.0, + "percentile99": 4.565333333333335, + "percentile999": 159.58346666667146, + "range": 213.86666666666667, + "samples": 264, + "stdev": 13.1630963406512, + "sum": 374.7999999999994 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 264, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.6166666666666667, + "max": 213.86666666666667, + "mean": 1.4196969696969697, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.7679245283018867, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6833333333333333, + "percentile90": 2.0, + "percentile99": 4.565333333333335, + "percentile999": 159.58346666667146, + "range": 213.86666666666667, + "samples": 264, + "stdev": 13.1630963406512, + "sum": 374.7999999999994 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.9499999999999997, + "max": 4006.8, + "mean": 41.11161616161616, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 102.39119496855346, + "non_zero_median": 15.366666666666667, + "percentile25": 0.0, + "percentile75": 2.9499999999999997, + "percentile90": 58.6, + "percentile99": 80.02600000000005, + "percentile999": 4005.8532, + "range": 4006.8, + "samples": 264, + "stdev": 347.69341792497886, + "sum": 10853.466666666665 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11598.833333333334, + "max": 451636.4, + "mean": 14966.68005050505, + "median": 14063.433333333334, + "min": 1284.7333333333333, + "non_zero_mean": 14966.68005050505, + "non_zero_median": 14063.433333333334, + "percentile25": 5858.383333333333, + "percentile75": 17457.216666666667, + "percentile90": 24070.09333333335, + "percentile99": 36038.38533333333, + "percentile999": 342429.3144666763, + "range": 450351.6666666667, + "samples": 264, + "stdev": 28229.42249581673, + "sum": 3951203.5333333327 + }, + "pg_stat_database_tup_inserted": { + "iqr": 9.266666666666667, + "max": 8005.333333333333, + "mean": 46.01161616161616, + "median": 2.7, + "min": 0.0, + "non_zero_mean": 55.466057838660575, + "non_zero_median": 4.266666666666667, + "percentile25": 1.0666666666666667, + "percentile75": 10.333333333333334, + "percentile90": 63.806666666666715, + "percentile99": 142.94066666666666, + "percentile999": 5937.855266666849, + "range": 8005.333333333333, + "samples": 264, + "stdev": 492.7912355740888, + "sum": 12147.066666666657 + }, + "pg_stat_database_tup_returned": { + "iqr": 34750.33333333333, + "max": 522636.6, + "mean": 45436.81035353535, + "median": 35085.566666666666, + "min": 3373.4, + "non_zero_mean": 45436.81035353535, + "non_zero_median": 35085.566666666666, + "percentile25": 19732.416666666668, + "percentile75": 54482.75, + "percentile90": 87311.38666666669, + "percentile99": 169009.4780000001, + "percentile999": 517210.69960000046, + "range": 519263.19999999995, + "samples": 264, + "stdev": 52507.5117979524, + "sum": 11995317.933333334 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.45, + "max": 67.66666666666667, + "mean": 10.315909090909091, + "median": 7.8, + "min": 0.13333333333333333, + "non_zero_mean": 10.315909090909091, + "non_zero_median": 7.8, + "percentile25": 3.3000000000000003, + "percentile75": 15.75, + "percentile90": 22.67333333333335, + "percentile99": 47.04200000000003, + "percentile999": 66.93026666666674, + "range": 67.53333333333333, + "samples": 264, + "stdev": 9.972568176111512, + "sum": 2723.399999999999 + }, + "pg_stat_database_xact_commit": { + "iqr": 125.48333333333332, + "max": 709.4, + "mean": 122.15479797979798, + "median": 90.4, + "min": 4.866666666666666, + "non_zero_mean": 122.15479797979798, + "non_zero_median": 90.4, + "percentile25": 37.68333333333334, + "percentile75": 163.16666666666666, + "percentile90": 276.1600000000001, + "percentile99": 561.8753333333334, + "percentile999": 696.2149333333344, + "range": 704.5333333333333, + "samples": 264, + "stdev": 123.16465042109895, + "sum": 32248.866666666672 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.12626262626262627, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.12626262626262627, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5333333333333333, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 264, + "stdev": 0.1437637774609752, + "sum": 33.33333333333334 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 264, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 27.083333333333336, + "max": 147.93333333333334, + "mean": 39.44142303517926, + "median": 31.833333333333336, + "min": 2.6666666666666665, + "non_zero_mean": 39.44142303517926, + "non_zero_median": 31.833333333333336, + "percentile25": 23.116666666666667, + "percentile75": 50.2, + "percentile90": 72.11333333333333, + "percentile99": 111.12533333333334, + "percentile999": 141.79666666666722, + "range": 145.26666666666668, + "samples": 264, + "stdev": 23.731028808006634, + "sum": 10412.53568128733 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2022-01-17T11:05:03.181621+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2022-01-17T09_07_03_881_0000", + "ended": "2022-01-17T12:11:01.784660+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.020028089999641452, + "iterations": 1, + "max": 0.5728287049996652, + "mean": 0.555148983333432, + "median": 0.5561530430004495, + "min": 0.5381410980007786, + "rounds": 15, + "stddev": 0.011569444845231003 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.019427612499612223, + "iterations": 1, + "max": 0.6244534630004637, + "mean": 0.5474914039333574, + "median": 0.5420199069994851, + "min": 0.5231382490001124, + "rounds": 15, + "stddev": 0.023638249402212527 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.008377144000405679, + "iterations": 1, + "max": 0.7903241779995369, + "mean": 0.4380987190998894, + "median": 0.40025216850017387, + "min": 0.38276204200064967, + "rounds": 10, + "stddev": 0.1240558567010419 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.00922997400039094, + "iterations": 1, + "max": 0.2789478300001065, + "mean": 0.2633955487699859, + "median": 0.2637128629999097, + "min": 0.2472716809998019, + "rounds": 100, + "stddev": 0.006921399353669834 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.01594107399978384, + "iterations": 1, + "max": 0.6027677189995302, + "mean": 0.48084922630998334, + "median": 0.478625900499992, + "min": 0.4505925479998041, + "rounds": 100, + "stddev": 0.02208625652272737 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.03713892199994007, + "iterations": 1, + "max": 1.5192889199997808, + "mean": 1.3737462408000283, + "median": 1.3576864359997671, + "min": 1.296006228000806, + "rounds": 10, + "stddev": 0.06517622960856595 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0015704339994044858, + "iterations": 1, + "max": 0.07592872899931535, + "mean": 0.07408323649990832, + "median": 0.07402588500053753, + "min": 0.0719112379993021, + "rounds": 14, + "stddev": 0.001143595894158591 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0029469060000337777, + "iterations": 1, + "max": 0.07899807800004055, + "mean": 0.07418061792869983, + "median": 0.07403531999989355, + "min": 0.07159871300063969, + "rounds": 14, + "stddev": 0.001998641894373956 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.01722143774941287, + "iterations": 1, + "max": 0.11617335500068293, + "mean": 0.10183242836377428, + "median": 0.09625372099981178, + "min": 0.09147861499968712, + "rounds": 11, + "stddev": 0.009361712588457964 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.00254414299979544, + "iterations": 1, + "max": 0.11232633599956898, + "mean": 0.10927541359978932, + "median": 0.1095177974998478, + "min": 0.10612930699971912, + "rounds": 10, + "stddev": 0.0017817666519153478 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.013752897000813391, + "iterations": 1, + "max": 0.38698398300039116, + "mean": 0.2946604004998335, + "median": 0.2765195944998595, + "min": 0.2697102279998944, + "rounds": 10, + "stddev": 0.04088685603140417 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.007016015999397496, + "iterations": 1, + "max": 0.22585593199983123, + "mean": 0.12180399380004019, + "median": 0.11067655550004929, + "min": 0.10517589399933058, + "rounds": 10, + "stddev": 0.03673315112829724 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.001822279000407434, + "iterations": 1, + "max": 0.07791065099991101, + "mean": 0.07215569378584925, + "median": 0.07196087750025981, + "min": 0.06997464200048853, + "rounds": 14, + "stddev": 0.0019577148997842304 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.018570741999610618, + "iterations": 1, + "max": 0.7601889190000293, + "mean": 0.6477345095997407, + "median": 0.6357617469993784, + "min": 0.6115234219996637, + "rounds": 10, + "stddev": 0.0419420804113872 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.003329678999762109, + "iterations": 1, + "max": 0.09725114999946527, + "mean": 0.0937440540909103, + "median": 0.0933716520003145, + "min": 0.08976555899971572, + "rounds": 11, + "stddev": 0.0023036701524606473 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.001862613999946916, + "iterations": 1, + "max": 0.10330477499974222, + "mean": 0.10154665330001081, + "median": 0.10172354350015667, + "min": 0.09971839600075327, + "rounds": 10, + "stddev": 0.0011813024198165552 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.0033128740005849977, + "iterations": 1, + "max": 0.19970822499999485, + "mean": 0.18097722809989136, + "median": 0.17840426200018555, + "min": 0.17611989899978653, + "rounds": 10, + "stddev": 0.006921375176360637 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.0018623699997988297, + "iterations": 1, + "max": 0.1866165670007831, + "mean": 0.08182286600003863, + "median": 0.07373206550028044, + "min": 0.07249519000015425, + "rounds": 14, + "stddev": 0.03018052955443828 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.003997781999714789, + "iterations": 1, + "max": 0.07995392699922377, + "mean": 0.0748971826426766, + "median": 0.07455262299936294, + "min": 0.07219366899971646, + "rounds": 14, + "stddev": 0.002334576582170925 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.006473434000326961, + "iterations": 1, + "max": 0.4032282080006553, + "mean": 0.38266955600001895, + "median": 0.37965017350006747, + "min": 0.3771775700006401, + "rounds": 10, + "stddev": 0.007882168611123085 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.017650061000495043, + "iterations": 1, + "max": 0.40052601100069296, + "mean": 0.3839359249998779, + "median": 0.38410805399962555, + "min": 0.36877555399951234, + "rounds": 10, + "stddev": 0.011005201789040471 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.00900020199969731, + "iterations": 1, + "max": 0.12334026500047912, + "mean": 0.1025839213000836, + "median": 0.09864339950036083, + "min": 0.0945316979996278, + "rounds": 10, + "stddev": 0.00858627940221335 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.002593359500679071, + "iterations": 1, + "max": 0.09324752600059583, + "mean": 0.08992287449988605, + "median": 0.0903231944998879, + "min": 0.08515569100018183, + "rounds": 12, + "stddev": 0.0022570562513802087 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.004752236999593151, + "iterations": 1, + "max": 0.18845671800045238, + "mean": 0.18347803400001794, + "median": 0.184844724499726, + "min": 0.1737895070000377, + "rounds": 10, + "stddev": 0.004965928945436073 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.003166650749562905, + "iterations": 1, + "max": 0.07774980999965919, + "mean": 0.07350825076928931, + "median": 0.07310397699984605, + "min": 0.06956490300035512, + "rounds": 13, + "stddev": 0.0024617591479361204 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.00854321400038316, + "iterations": 1, + "max": 0.23608909199992922, + "mean": 0.21393094669992935, + "median": 0.21291644749953775, + "min": 0.20150219199967978, + "rounds": 10, + "stddev": 0.009859068765554972 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0014518465004584868, + "iterations": 1, + "max": 0.08052900999973645, + "mean": 0.0780704526921629, + "median": 0.07820336399981898, + "min": 0.0758195379994504, + "rounds": 13, + "stddev": 0.0012064763182357942 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.010791739000524103, + "iterations": 1, + "max": 0.37653003400009766, + "mean": 0.3588422305999302, + "median": 0.35943447999989075, + "min": 0.3461373439995441, + "rounds": 10, + "stddev": 0.008901228162831966 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.011064609999266395, + "iterations": 1, + "max": 0.296456758000204, + "mean": 0.2424710612001036, + "median": 0.23572021750032945, + "min": 0.23040979300003528, + "rounds": 10, + "stddev": 0.01963620593371489 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.009753136000654195, + "iterations": 1, + "max": 0.17829142800019326, + "mean": 0.16769246109988672, + "median": 0.16616190500008088, + "min": 0.16120032699927833, + "rounds": 10, + "stddev": 0.005822996288041467 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.13726683300046716, + "iterations": 1, + "max": 0.2693468589995973, + "mean": 0.17427274900001066, + "median": 0.16937419650002994, + "min": 0.10191653100082476, + "rounds": 10, + "stddev": 0.07377843739752883 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.004423753999617475, + "iterations": 1, + "max": 0.0792718259999674, + "mean": 0.0740869335714446, + "median": 0.07336314449958081, + "min": 0.0707996679993812, + "rounds": 14, + "stddev": 0.0026494903941668042 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.005980057000670058, + "iterations": 1, + "max": 0.10364014500009944, + "mean": 0.09633655659990836, + "median": 0.09583983599986823, + "min": 0.08818751099988731, + "rounds": 10, + "stddev": 0.0046481525906162426 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0021673870005542994, + "iterations": 1, + "max": 0.07556344100066781, + "mean": 0.07321290678575158, + "median": 0.07344824300025721, + "min": 0.07151755799986859, + "rounds": 14, + "stddev": 0.0013070155070671536 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.0028449010005715536, + "iterations": 1, + "max": 0.12563000899990584, + "mean": 0.12339506709986381, + "median": 0.1241850454998712, + "min": 0.11749103700003616, + "rounds": 10, + "stddev": 0.002607560897984553 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0020707150006273878, + "iterations": 1, + "max": 0.07609333100026561, + "mean": 0.07362278821431444, + "median": 0.0734118190002846, + "min": 0.07196576800015464, + "rounds": 14, + "stddev": 0.0012630323030250264 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.022334572999170632, + "iterations": 1, + "max": 0.7324876110005789, + "mean": 0.6305763556000784, + "median": 0.6255660689998876, + "min": 0.598377328999959, + "rounds": 10, + "stddev": 0.03792078282478849 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.028998809999393416, + "iterations": 1, + "max": 0.7628916300000128, + "mean": 0.6426004654999815, + "median": 0.6313161230000333, + "min": 0.6000336490005793, + "rounds": 10, + "stddev": 0.0460437185381653 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.00825820099998964, + "iterations": 1, + "max": 0.47468506200038973, + "mean": 0.3002633734000483, + "median": 0.28303885700006504, + "min": 0.2677977830007876, + "rounds": 10, + "stddev": 0.06174475250805197 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0017719479997140297, + "iterations": 1, + "max": 0.0769452939994153, + "mean": 0.07135763846666426, + "median": 0.07091686700005084, + "min": 0.06931311100015591, + "rounds": 15, + "stddev": 0.0019351724883754724 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.003602025999498437, + "iterations": 1, + "max": 0.08803180900031293, + "mean": 0.07517917364306673, + "median": 0.07461086500006786, + "min": 0.07047513500037894, + "rounds": 14, + "stddev": 0.004505508005413143 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0055775107507543, + "iterations": 1, + "max": 0.08451207500002056, + "mean": 0.0791416195383695, + "median": 0.07874232799986203, + "min": 0.0748220069999661, + "rounds": 13, + "stddev": 0.0034160064748101843 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0024852025001109723, + "iterations": 1, + "max": 0.07776074799949129, + "mean": 0.07502193223081122, + "median": 0.0747375529999772, + "min": 0.07214831199962646, + "rounds": 13, + "stddev": 0.001774233989298877 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.016828886999064707, + "iterations": 1, + "max": 0.31384564300060447, + "mean": 0.2988391432001663, + "median": 0.29817698250008107, + "min": 0.2827836180003942, + "rounds": 10, + "stddev": 0.010778522830349361 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.01538182100011909, + "iterations": 1, + "max": 0.23900958899957914, + "mean": 0.22653015529995174, + "median": 0.22300524299953395, + "min": 0.218764039999769, + "rounds": 10, + "stddev": 0.008189214696175246 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.017638639000324474, + "iterations": 1, + "max": 1.0382629730002009, + "mean": 0.9417013882000902, + "median": 0.9339158204998057, + "min": 0.907555789000071, + "rounds": 10, + "stddev": 0.036016937559423146 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.09141024699965783, + "iterations": 1, + "max": 8.876986903000216, + "mean": 8.722236137800156, + "median": 8.709128168500229, + "min": 8.601156329000332, + "rounds": 10, + "stddev": 0.0797853945201078 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.022628947999692173, + "iterations": 1, + "max": 1.1466376830003355, + "mean": 1.0472734736002167, + "median": 1.0385964830002195, + "min": 0.9943375440007003, + "rounds": 10, + "stddev": 0.04118224255131779 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.2766883539998162, + "iterations": 1, + "max": 9.427612444999795, + "mean": 9.127242040700002, + "median": 9.068325068999911, + "min": 8.987148866999632, + "rounds": 10, + "stddev": 0.1563910981788554 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.46145363400046335, + "iterations": 1, + "max": 1.8836889169997448, + "mean": 1.3618084565000572, + "median": 1.3127606580001157, + "min": 1.0394562680003219, + "rounds": 10, + "stddev": 0.29949835178568696 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.01603813900055684, + "iterations": 1, + "max": 0.42012740399968607, + "mean": 0.4057828143998449, + "median": 0.4100062425000033, + "min": 0.3855980649996127, + "rounds": 10, + "stddev": 0.011861685679814602 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.34428961199955666, + "iterations": 1, + "max": 11.885299595998731, + "mean": 11.596220602199718, + "median": 11.527903779498956, + "min": 11.441466794000007, + "rounds": 10, + "stddev": 0.17160840364364952 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.011553919001016766, + "iterations": 1, + "max": 0.4120622119999098, + "mean": 0.4035965939001471, + "median": 0.4028839715006143, + "min": 0.396283174999553, + "rounds": 10, + "stddev": 0.005843647866799281 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.2717983210004604, + "iterations": 1, + "max": 0.9734777369994845, + "mean": 0.6456657559998348, + "median": 0.6309915474998888, + "min": 0.4275872309999613, + "rounds": 10, + "stddev": 0.17969343584632938 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.39048719799939136, + "iterations": 1, + "max": 11.83256133299983, + "mean": 11.556998105599813, + "median": 11.55471604649938, + "min": 11.337488600000142, + "rounds": 10, + "stddev": 0.19768186284135766 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.08654847200068616, + "iterations": 1, + "max": 8.036541306999425, + "mean": 7.7661551662999955, + "median": 7.755475362499965, + "min": 7.606166771000062, + "rounds": 10, + "stddev": 0.1169599668583585 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 250.03429250000045, + "mean": 250.03429250000045, + "median": 250.03429250000045, + "min": 250.03429250000045, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 399.95078974699936, + "mean": 399.95078974699936, + "median": 399.95078974699936, + "min": 399.95078974699936, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.00610992299971258, + "iterations": 1, + "max": 0.26911965100043744, + "mean": 0.256854208599907, + "median": 0.25753903299982994, + "min": 0.247526402999938, + "rounds": 10, + "stddev": 0.005994412634941436 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.01041166799950588, + "iterations": 1, + "max": 0.7758289069997772, + "mean": 0.6583440800998688, + "median": 0.648819653999908, + "min": 0.6260642609995557, + "rounds": 10, + "stddev": 0.0420682071080228 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.16593407000073057, + "iterations": 1, + "max": 10.086649450999175, + "mean": 9.701163009300036, + "median": 9.69508196750121, + "min": 9.386123031001262, + "rounds": 10, + "stddev": 0.18236095961168783 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.5186964149997948, + "iterations": 1, + "max": 3.6811503680000897, + "mean": 3.181690921199788, + "median": 3.0535566119997384, + "min": 2.8371097560002454, + "rounds": 10, + "stddev": 0.3162660569822864 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.44546175100003893, + "iterations": 1, + "max": 5.24687682000058, + "mean": 2.306844125900079, + "median": 1.994320196999979, + "min": 1.4720939749995523, + "rounds": 10, + "stddev": 1.0826182744420925 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.9614943489996222, + "iterations": 1, + "max": 26.35735353599921, + "mean": 25.21321058539961, + "median": 25.14477349650042, + "min": 24.456878498998776, + "rounds": 10, + "stddev": 0.7101691345986721 + } + }, + "started": "2022-01-17T10:54:48.770375+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-chatty.json new file mode 100644 index 00000000..b7659518 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-chatty.json @@ -0,0 +1,636 @@ +{ + "ended": "2022-01-17T10:08:25.392437+00:00", + "id": "run-2022-01-17T09_07_03_881_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 13098.766666666666, + "max": 51420.66666666667, + "mean": 31476.848958333332, + "median": 38118.2, + "min": 1604.6666666666665, + "non_zero_mean": 31476.848958333332, + "non_zero_median": 38118.2, + "percentile25": 26402.983333333334, + "percentile75": 39501.75, + "percentile90": 40791.78, + "percentile99": 45455.994666666644, + "percentile999": 50824.199466666716, + "range": 49816.00000000001, + "samples": 64, + "stdev": 13057.930615134383, + "sum": 2014518.3333333328 + } + }, + "cpu": { + "system": { + "iqr": 0.2726666666666674, + "max": 1.0873333333333335, + "mean": 0.8298958333333334, + "median": 0.9933333333333327, + "min": 0.039999999999998676, + "non_zero_mean": 0.8298958333333334, + "non_zero_median": 0.9933333333333327, + "percentile25": 0.7420000000000003, + "percentile75": 1.0146666666666677, + "percentile90": 1.0277999999999994, + "percentile99": 1.0680133333333324, + "percentile999": 1.0854013333333334, + "range": 1.0473333333333348, + "samples": 64, + "stdev": 0.3106033093591124, + "sum": 53.113333333333365 + }, + "user": { + "iqr": 1.9308333333333252, + "max": 6.579333333333314, + "mean": 5.087229166666667, + "median": 6.365333333333338, + "min": 0.275999999999984, + "non_zero_mean": 5.087229166666667, + "non_zero_median": 6.365333333333338, + "percentile25": 4.487166666666672, + "percentile75": 6.4179999999999975, + "percentile90": 6.466333333333333, + "percentile99": 6.574713333333318, + "percentile999": 6.578871333333314, + "range": 6.30333333333333, + "samples": 64, + "stdev": 2.0780753176952245, + "sum": 325.5826666666667 + } + }, + "entropy_available_bits": { + "iqr": 7.933333333333334, + "max": 424.0, + "mean": 23.852083333333333, + "median": 7.166666666666666, + "min": 0.0, + "non_zero_mean": 33.18550724637681, + "non_zero_median": 7.733333333333333, + "percentile25": 0.0, + "percentile75": 7.933333333333334, + "percentile90": 13.833333333333336, + "percentile99": 423.07599999999996, + "percentile999": 423.9076, + "range": 424.0, + "samples": 64, + "stdev": 84.89259982643367, + "sum": 1526.5333333333328 + }, + "memory": { + "used": { + "iqr": 1372314624.0, + "max": 7461224448.0, + "mean": 6445757952.0, + "median": 6926913536.0, + "min": 3461791744.0, + "non_zero_mean": 6445757952.0, + "non_zero_median": 6926913536.0, + "percentile25": 5714484224.0, + "percentile75": 7086798848.0, + "percentile90": 7346574540.8, + "percentile99": 7455260958.72, + "percentile999": 7460628099.0720005, + "range": 3999432704.0, + "samples": 64, + "stdev": 975324785.026062, + "sum": 412528508928.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 4.0, + "mean": 0.09583333333333334, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.5333333333333334, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.5719999999999943, + "percentile999": 3.857200000000011, + "range": 4.0, + "samples": 64, + "stdev": 0.5419769156267732, + "sum": 6.133333333333334 + } + }, + "writes_completed": { + "total": { + "iqr": 2288.1500000000005, + "max": 5009.8, + "mean": 3499.7239583333335, + "median": 4491.633333333333, + "min": 14.8, + "non_zero_mean": 3499.7239583333335, + "non_zero_median": 4491.633333333333, + "percentile25": 2415.033333333333, + "percentile75": 4703.183333333333, + "percentile90": 4778.653333333333, + "percentile99": 4998.418000000001, + "percentile999": 5008.6618, + "range": 4995.0, + "samples": 64, + "stdev": 1683.8623675305216, + "sum": 223982.33333333328 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 224.58333333333326, + "max": 2214.5333333333333, + "mean": 1628.2791666666667, + "median": 1701.3, + "min": 592.2666666666667, + "non_zero_mean": 1628.2791666666667, + "non_zero_median": 1701.3, + "percentile25": 1565.1166666666668, + "percentile75": 1789.7, + "percentile90": 1934.18, + "percentile99": 2177.909333333333, + "percentile999": 2210.8709333333336, + "range": 1622.2666666666667, + "samples": 64, + "stdev": 330.13303114648835, + "sum": 104209.86666666662 + } + }, + "cpu": { + "system": { + "iqr": 0.0351666666666669, + "max": 0.244, + "mean": 0.14144791666666667, + "median": 0.15366666666666662, + "min": 0.023333333333332956, + "non_zero_mean": 0.14144791666666667, + "non_zero_median": 0.15366666666666662, + "percentile25": 0.12966666666666654, + "percentile75": 0.16483333333333344, + "percentile90": 0.18146666666666675, + "percentile99": 0.2259399999999998, + "percentile999": 0.2421940000000001, + "range": 0.22066666666666704, + "samples": 64, + "stdev": 0.04426145737938873, + "sum": 9.052666666666667 + }, + "user": { + "iqr": 0.06449999999999934, + "max": 0.5973333333333334, + "mean": 0.23098958333333333, + "median": 0.1786666666666669, + "min": 0.034000000000000107, + "non_zero_mean": 0.23098958333333333, + "non_zero_median": 0.1786666666666669, + "percentile25": 0.1623333333333335, + "percentile75": 0.22683333333333283, + "percentile90": 0.5138666666666666, + "percentile99": 0.5759133333333334, + "percentile999": 0.5951913333333335, + "range": 0.5633333333333332, + "samples": 64, + "stdev": 0.14424958735601626, + "sum": 14.783333333333335 + } + }, + "entropy_available_bits": { + "iqr": 1.4666666666666668, + "max": 190.13333333333333, + "mean": 10.1625, + "median": 2.166666666666667, + "min": 0.4, + "non_zero_mean": 10.1625, + "non_zero_median": 2.166666666666667, + "percentile25": 1.4, + "percentile75": 2.8666666666666667, + "percentile90": 4.746666666666668, + "percentile99": 181.31333333333328, + "percentile999": 189.25133333333338, + "range": 189.73333333333332, + "samples": 64, + "stdev": 36.1446103470885, + "sum": 650.3999999999997 + }, + "memory": { + "used": { + "iqr": 43462656.0, + "max": 488419328.0, + "mean": 440993536.0, + "median": 456585216.0, + "min": 360681472.0, + "non_zero_mean": 440993536.0, + "non_zero_median": 456585216.0, + "percentile25": 421297152.0, + "percentile75": 464759808.0, + "percentile90": 469857075.2, + "percentile99": 483591249.91999996, + "percentile999": 487936520.19200003, + "range": 127737856.0, + "samples": 64, + "stdev": 34117874.35976793, + "sum": 28223586304.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1708987.7333333334, + "max": 3376196.2666666666, + "mean": 818645.3333333334, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2095732.0533333332, + "non_zero_median": 2227131.7333333334, + "percentile25": 0.0, + "percentile75": 1708987.7333333334, + "percentile90": 3273905.493333333, + "percentile99": 3364498.0906666666, + "percentile999": 3375026.449066667, + "range": 3376196.2666666666, + "samples": 64, + "stdev": 1256816.964922761, + "sum": 52393301.333333336 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.06974875, + "max": 0.742542, + "mean": 0.089295515625, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.21980434615384617, + "non_zero_median": 0.2216195, + "percentile25": 0.0, + "percentile75": 0.06974875, + "percentile90": 0.35016800000000003, + "percentile99": 0.7154450699999999, + "percentile999": 0.7398323070000002, + "range": 0.742542, + "samples": 64, + "stdev": 0.17127095281583565, + "sum": 5.714913 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.1412525, + "max": 0.771911, + "mean": 0.164219671875, + "median": 0.0, + "min": -7e-05, + "non_zero_mean": 0.37535925, + "non_zero_median": 0.388926, + "percentile25": 0.0, + "percentile75": 0.1412525, + "percentile90": 0.7152221, + "percentile99": 0.76687226, + "percentile999": 0.771407126, + "range": 0.771981, + "samples": 64, + "stdev": 0.28667607568440595, + "sum": 10.510059000000002 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 11.266666666666667, + "mean": 0.321875, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 6.866666666666667, + "non_zero_median": 8.066666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9.250666666666659, + "percentile999": 11.065066666666683, + "range": 11.266666666666667, + "samples": 64, + "stdev": 1.7227824869537354, + "sum": 20.6 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9994.466666666667, + "mean": 467.78020833333335, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9979.311111111112, + "non_zero_median": 9985.866666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9989.048666666667, + "percentile999": 9993.924866666666, + "range": 9994.466666666667, + "samples": 64, + "stdev": 2126.015432845102, + "sum": 29937.933333333334 + }, + "pg_stat_database_blks_hit": { + "iqr": 3607.6500000000015, + "max": 39472.0, + "mean": 21908.436458333334, + "median": 23112.166666666664, + "min": 5840.8, + "non_zero_mean": 21908.436458333334, + "non_zero_median": 23112.166666666664, + "percentile25": 20868.0, + "percentile75": 24475.65, + "percentile90": 25637.333333333336, + "percentile99": 35632.359999999986, + "percentile999": 39088.03600000003, + "range": 33631.2, + "samples": 64, + "stdev": 5706.643877024288, + "sum": 1402139.9333333333 + }, + "pg_stat_database_blks_read": { + "iqr": 72.76666666666667, + "max": 159.86666666666667, + "mean": 99.88020833333333, + "median": 131.33333333333331, + "min": 0.06666666666666667, + "non_zero_mean": 99.88020833333333, + "non_zero_median": 131.33333333333331, + "percentile25": 63.91666666666667, + "percentile75": 136.68333333333334, + "percentile90": 139.67333333333332, + "percentile99": 151.59266666666662, + "percentile999": 159.03926666666675, + "range": 159.8, + "samples": 64, + "stdev": 50.422500198171136, + "sum": 6392.333333333333 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 72.76666666666667, + "max": 159.86666666666667, + "mean": 99.88020833333333, + "median": 131.33333333333331, + "min": 0.06666666666666667, + "non_zero_mean": 99.88020833333333, + "non_zero_median": 131.33333333333331, + "percentile25": 63.91666666666667, + "percentile75": 136.68333333333334, + "percentile90": 139.67333333333332, + "percentile99": 151.59266666666662, + "percentile999": 159.03926666666675, + "range": 159.8, + "samples": 64, + "stdev": 50.422500198171136, + "sum": 6392.333333333333 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.016666666666666666, + "max": 1.4666666666666666, + "mean": 0.2125, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.85, + "non_zero_median": 0.9666666666666667, + "percentile25": 0.0, + "percentile75": 0.016666666666666666, + "percentile90": 1.0, + "percentile99": 1.4246666666666663, + "percentile999": 1.4624666666666668, + "range": 1.4666666666666666, + "samples": 64, + "stdev": 0.442675049324595, + "sum": 13.6 + }, + "pg_stat_database_tup_fetched": { + "iqr": 1983.1499999999996, + "max": 25157.2, + "mean": 7256.135416666667, + "median": 5615.5, + "min": 2944.0666666666666, + "non_zero_mean": 7256.135416666667, + "non_zero_median": 5615.5, + "percentile25": 5089.666666666667, + "percentile75": 7072.816666666667, + "percentile90": 12381.066666666668, + "percentile99": 24333.411999999997, + "percentile999": 25074.821200000006, + "range": 22213.133333333335, + "samples": 64, + "stdev": 4325.80978040197, + "sum": 464392.66666666657 + }, + "pg_stat_database_tup_inserted": { + "iqr": 386.84999999999997, + "max": 829.7333333333333, + "mean": 531.0354166666667, + "median": 697.1333333333333, + "min": 1.3333333333333333, + "non_zero_mean": 531.0354166666667, + "non_zero_median": 697.1333333333333, + "percentile25": 337.4166666666667, + "percentile75": 724.2666666666667, + "percentile90": 741.7666666666667, + "percentile99": 802.7693333333332, + "percentile999": 827.0369333333335, + "range": 828.4, + "samples": 64, + "stdev": 266.1943640109928, + "sum": 33986.26666666666 + }, + "pg_stat_database_tup_returned": { + "iqr": 2561.083333333332, + "max": 29464.066666666666, + "mean": 9112.660416666668, + "median": 7002.7, + "min": 4608.866666666667, + "non_zero_mean": 9112.660416666668, + "non_zero_median": 7002.7, + "percentile25": 6174.516666666666, + "percentile75": 8735.599999999999, + "percentile90": 17507.466666666667, + "percentile99": 29369.524666666664, + "percentile999": 29454.612466666666, + "range": 24855.199999999997, + "samples": 64, + "stdev": 5545.0121781374155, + "sum": 583210.2666666665 + }, + "pg_stat_database_tup_updated": { + "iqr": 6.483333333333333, + "max": 136.66666666666666, + "mean": 10.194791666666667, + "median": 0.8666666666666667, + "min": 0.0, + "non_zero_mean": 12.082716049382716, + "non_zero_median": 1.5333333333333334, + "percentile25": 0.25, + "percentile75": 6.733333333333333, + "percentile90": 24.813333333333354, + "percentile99": 127.88866666666662, + "percentile999": 135.78886666666673, + "range": 136.66666666666666, + "samples": 64, + "stdev": 25.786474462625563, + "sum": 652.4666666666665 + }, + "pg_stat_database_xact_commit": { + "iqr": 10.350000000000001, + "max": 143.86666666666667, + "mean": 59.067708333333336, + "median": 50.53333333333333, + "min": 18.2, + "non_zero_mean": 59.067708333333336, + "non_zero_median": 50.53333333333333, + "percentile25": 46.71666666666667, + "percentile75": 57.06666666666667, + "percentile90": 99.80000000000001, + "percentile99": 142.35466666666667, + "percentile999": 143.71546666666669, + "range": 125.66666666666667, + "samples": 64, + "stdev": 26.67191348432549, + "sum": 3780.333333333332 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 64, + "stdev": 0.0, + "sum": 4.266666666666671 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.2, + "mean": 0.022916666666666665, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7333333333333333, + "non_zero_median": 0.7333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.6119999999999977, + "percentile999": 1.1412000000000044, + "range": 1.2, + "samples": 64, + "stdev": 0.1531417021989143, + "sum": 1.4666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 25.466666666666654, + "max": 211.33333333333334, + "mean": 79.31354166666667, + "median": 79.63333333333333, + "min": 9.066666666666666, + "non_zero_mean": 79.31354166666667, + "non_zero_median": 79.63333333333333, + "percentile25": 68.10000000000001, + "percentile75": 93.56666666666666, + "percentile90": 109.12666666666668, + "percentile99": 157.9933333333331, + "percentile999": 205.99933333333374, + "range": 202.26666666666668, + "samples": 64, + "stdev": 30.921185992225215, + "sum": 5076.066666666667 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,5,4,6,7,8,11,10,9,14,18,21,19,23,24,22,20,25,26,27,29,28,30,32,33,31,35,34,36,37,38,41,40,39,43,45,44,42,46,47,48,49,51,54,53,50,52,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-17T09:49:45.043393Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "FAIL", + "results": { + "created": "2022-01-17T09:49:46.179007Z 2022-01-17T09:49:46.067358Z 2022-01-17T09:49:45.754218Z 2022-01-17T09:49:45.253918Z 2022-01-17T09:49:45.168578Z 2022-01-17T09:49:45.161211Z 2022-01-17T09:49:45.043393Z 2022-01-17T09:49:46.950032Z 2022-01-17T09:49:46.312152Z 2022-01-17T09:49:46.204090Z 2022-01-17T09:53:58.324520Z 2022-01-17T09:53:58.298690Z 2022-01-17T09:53:57.893038Z 2022-01-17T09:53:57.835945Z 2022-01-17T09:53:57.769964Z 2022-01-17T09:53:57.756986Z 2022-01-17T09:53:57.753494Z 2022-01-17T09:53:57.743592Z 2022-01-17T09:53:57.743170Z 2022-01-17T09:53:57.653056Z 2022-01-17T09:57:03.695727Z 2022-01-17T09:57:03.009180Z 2022-01-17T09:57:02.642341Z 2022-01-17T09:57:02.622871Z 2022-01-17T09:57:02.579446Z 2022-01-17T09:57:02.558252Z 2022-01-17T09:57:02.553684Z 2022-01-17T09:57:02.541725Z 2022-01-17T09:57:02.439063Z 2022-01-17T09:57:02.430003Z 2022-01-17T10:00:09.643302Z 2022-01-17T10:00:08.949824Z 2022-01-17T10:00:08.563189Z 2022-01-17T10:00:08.473838Z 2022-01-17T10:00:08.468814Z 2022-01-17T10:00:08.467929Z 2022-01-17T10:00:08.439854Z 2022-01-17T10:00:08.362082Z 2022-01-17T10:00:08.359976Z 2022-01-17T10:00:08.333860Z 2022-01-17T10:03:06.609787Z 2022-01-17T10:03:06.086587Z 2022-01-17T10:03:05.862034Z 2022-01-17T10:03:05.588390Z 2022-01-17T10:03:05.582498Z 2022-01-17T10:03:05.578347Z 2022-01-17T10:03:05.541973Z 2022-01-17T10:03:05.501092Z 2022-01-17T10:03:05.438995Z 2022-01-17T10:03:05.435332Z", + "created_diff": 1.2655448, + "ended": "2022-01-17T10:05:43.209009Z", + "error_job_ids": " []", + "event_first": "2022-01-17T09:49:58.583023Z 2022-01-17T09:51:32.606015Z 2022-01-17T09:49:56.469221Z 2022-01-17T09:51:13.052973Z 2022-01-17T09:49:51.394617Z 2022-01-17T09:50:34.931698Z 2022-01-17T09:49:51.317449Z 2022-01-17T09:50:46.453823Z 2022-01-17T09:50:00.607767Z 2022-01-17T09:50:58.768040Z 2022-01-17T09:54:18.899879Z 2022-01-17T09:54:17.338827Z 2022-01-17T09:54:06.314585Z 2022-01-17T09:54:07.969827Z 2022-01-17T09:54:05.042970Z 2022-01-17T09:54:05.795544Z 2022-01-17T09:54:03.791097Z 2022-01-17T09:54:04.108054Z 2022-01-17T09:54:04.549943Z 2022-01-17T09:54:03.856974Z 2022-01-17T09:57:13.895399Z 2022-01-17T09:57:18.679811Z 2022-01-17T09:57:16.694309Z 2022-01-17T09:57:10.061190Z 2022-01-17T09:57:10.662200Z 2022-01-17T09:57:12.508503Z 2022-01-17T09:57:10.834059Z 2022-01-17T09:57:10.566492Z 2022-01-17T09:57:09.159493Z 2022-01-17T09:57:08.774990Z 2022-01-17T10:00:22.621959Z 2022-01-17T10:00:24.150724Z 2022-01-17T10:00:21.851268Z 2022-01-17T10:00:20.311806Z 2022-01-17T10:00:17.868739Z 2022-01-17T10:00:15.234219Z 2022-01-17T10:00:15.065202Z 2022-01-17T10:00:15.976701Z 2022-01-17T10:00:15.320125Z 2022-01-17T10:00:16.232214Z 2022-01-17T10:03:19.848916Z 2022-01-17T10:03:20.243256Z 2022-01-17T10:03:18.022243Z 2022-01-17T10:03:15.435583Z 2022-01-17T10:03:13.429033Z 2022-01-17T10:03:15.183443Z 2022-01-17T10:03:14.463993Z 2022-01-17T10:03:13.657646Z 2022-01-17T10:03:11.689742Z 2022-01-17T10:03:12.147207Z", + "event_last": "2022-01-17T09:52:33.621035Z 2022-01-17T09:53:42.011635Z 2022-01-17T09:52:30.474996Z 2022-01-17T09:53:29.688544Z 2022-01-17T09:52:25.047790Z 2022-01-17T09:53:02.152268Z 2022-01-17T09:52:24.079182Z 2022-01-17T09:53:12.545167Z 2022-01-17T09:52:33.312765Z 2022-01-17T09:53:19.737646Z 2022-01-17T09:56:38.839377Z 2022-01-17T09:56:35.967300Z 2022-01-17T09:56:29.198589Z 2022-01-17T09:56:30.513824Z 2022-01-17T09:56:28.086764Z 2022-01-17T09:56:29.256540Z 2022-01-17T09:56:27.122486Z 2022-01-17T09:56:28.274258Z 2022-01-17T09:56:27.113642Z 2022-01-17T09:56:26.722224Z 2022-01-17T09:59:36.505638Z 2022-01-17T09:59:42.219636Z 2022-01-17T09:59:38.541290Z 2022-01-17T09:59:35.651269Z 2022-01-17T09:59:33.603996Z 2022-01-17T09:59:34.691338Z 2022-01-17T09:59:34.271720Z 2022-01-17T09:59:36.367875Z 2022-01-17T09:59:33.141905Z 2022-01-17T09:59:32.931114Z 2022-01-17T10:02:45.015852Z 2022-01-17T10:02:45.270515Z 2022-01-17T10:02:43.946059Z 2022-01-17T10:02:41.652446Z 2022-01-17T10:02:42.065728Z 2022-01-17T10:02:39.247444Z 2022-01-17T10:02:38.613818Z 2022-01-17T10:02:39.158498Z 2022-01-17T10:02:38.809847Z 2022-01-17T10:02:39.836398Z 2022-01-17T10:05:42.468313Z 2022-01-17T10:05:43.209009Z 2022-01-17T10:05:42.243288Z 2022-01-17T10:05:38.705854Z 2022-01-17T10:05:36.663074Z 2022-01-17T10:05:38.883728Z 2022-01-17T10:05:40.431067Z 2022-01-17T10:05:36.799372Z 2022-01-17T10:05:37.043363Z 2022-01-17T10:05:36.105010Z", + "failed_job_ids": " []", + "finished": "2022-01-17T09:52:37.198262Z 2022-01-17T09:53:43.753259Z 2022-01-17T09:52:34.571046Z 2022-01-17T09:53:33.279381Z 2022-01-17T09:52:29.116115Z 2022-01-17T09:53:06.182603Z 2022-01-17T09:52:27.904384Z 2022-01-17T09:53:16.051467Z 2022-01-17T09:52:37.457515Z 2022-01-17T09:53:23.497483Z 2022-01-17T09:56:40.645245Z 2022-01-17T09:56:39.344936Z 2022-01-17T09:56:33.193875Z 2022-01-17T09:56:34.481794Z 2022-01-17T09:56:31.908570Z 2022-01-17T09:56:33.108159Z 2022-01-17T09:56:31.465880Z 2022-01-17T09:56:31.834648Z 2022-01-17T09:56:30.595428Z 2022-01-17T09:56:30.734678Z 2022-01-17T09:59:40.082562Z 2022-01-17T09:59:44.460367Z 2022-01-17T09:59:42.338423Z 2022-01-17T09:59:39.888212Z 2022-01-17T09:59:37.745938Z 2022-01-17T09:59:38.676483Z 2022-01-17T09:59:37.687723Z 2022-01-17T09:59:40.292049Z 2022-01-17T09:59:37.143985Z 2022-01-17T09:59:37.163640Z 2022-01-17T10:02:48.769844Z 2022-01-17T10:02:49.130791Z 2022-01-17T10:02:47.551311Z 2022-01-17T10:02:45.228152Z 2022-01-17T10:02:45.863340Z 2022-01-17T10:02:43.390046Z 2022-01-17T10:02:42.243886Z 2022-01-17T10:02:42.599879Z 2022-01-17T10:02:42.144303Z 2022-01-17T10:02:43.270102Z 2022-01-17T10:05:46.484174Z 2022-01-17T10:05:47.211194Z 2022-01-17T10:05:46.311924Z 2022-01-17T10:05:42.175869Z 2022-01-17T10:05:40.861065Z 2022-01-17T10:05:42.919363Z 2022-01-17T10:05:44.780845Z 2022-01-17T10:05:40.631777Z 2022-01-17T10:05:40.436601Z 2022-01-17T10:05:40.234556Z", + "finished_diff": 21.43564, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 236.066267, + "mean": 161.92310289999995, + "min": 151.863077 + }, + "jobs_events_duration": { + "max": 155.038012, + "mean": 143.90835410000005, + "min": 129.40562 + }, + "jobs_events_lag": { + "max": -1.741624, + "mean": -3.7236527199999996, + "min": -4.349778 + }, + "jobs_waiting": { + "max": 17.538737, + "mean": 2.013231600000001, + "min": 0.922071 + }, + "modified": "2022-01-17T09:49:47.334821Z 2022-01-17T09:49:47.299486Z 2022-01-17T09:49:47.257618Z 2022-01-17T09:49:45.794470Z 2022-01-17T09:49:45.759100Z 2022-01-17T09:49:45.716116Z 2022-01-17T09:49:45.682299Z 2022-01-17T09:50:04.283626Z 2022-01-17T09:49:47.408152Z 2022-01-17T09:49:47.373868Z 2022-01-17T09:54:06.631890Z 2022-01-17T09:54:06.561999Z 2022-01-17T09:53:58.480358Z 2022-01-17T09:53:58.445304Z 2022-01-17T09:53:58.401979Z 2022-01-17T09:53:58.366164Z 2022-01-17T09:53:58.335750Z 2022-01-17T09:53:58.305357Z 2022-01-17T09:53:58.270484Z 2022-01-17T09:53:58.233036Z 2022-01-17T09:57:04.774562Z 2022-01-17T09:57:04.736827Z 2022-01-17T09:57:04.699212Z 2022-01-17T09:57:03.180989Z 2022-01-17T09:57:03.150080Z 2022-01-17T09:57:03.113609Z 2022-01-17T09:57:03.081238Z 2022-01-17T09:57:03.047053Z 2022-01-17T09:57:03.014213Z 2022-01-17T09:57:02.982495Z 2022-01-17T10:00:10.795776Z 2022-01-17T10:00:10.761458Z 2022-01-17T10:00:10.673353Z 2022-01-17T10:00:09.148602Z 2022-01-17T10:00:09.116741Z 2022-01-17T10:00:09.079930Z 2022-01-17T10:00:09.041875Z 2022-01-17T10:00:09.009521Z 2022-01-17T10:00:08.964804Z 2022-01-17T10:00:08.923125Z 2022-01-17T10:03:07.581692Z 2022-01-17T10:03:07.543802Z 2022-01-17T10:03:07.509057Z 2022-01-17T10:03:06.223905Z 2022-01-17T10:03:06.192319Z 2022-01-17T10:03:06.159642Z 2022-01-17T10:03:06.128624Z 2022-01-17T10:03:06.096193Z 2022-01-17T10:03:06.064297Z 2022-01-17T10:03:06.028614Z", + "modified_diff": 6.443595400000001, + "started": "2022-01-17T09:49:47.787883Z 2022-01-17T09:49:47.686992Z 2022-01-17T09:49:47.596961Z 2022-01-17T09:49:46.396516Z 2022-01-17T09:49:46.181738Z 2022-01-17T09:49:46.213518Z 2022-01-17T09:49:46.006804Z 2022-01-17T09:50:04.488769Z 2022-01-17T09:49:48.093470Z 2022-01-17T09:49:47.867956Z 2022-01-17T09:54:07.643521Z 2022-01-17T09:54:07.378306Z 2022-01-17T09:53:59.290480Z 2022-01-17T09:53:59.240104Z 2022-01-17T09:53:59.075319Z 2022-01-17T09:53:59.018204Z 2022-01-17T09:53:58.876949Z 2022-01-17T09:53:58.805512Z 2022-01-17T09:53:58.732351Z 2022-01-17T09:53:58.654932Z 2022-01-17T09:57:05.227219Z 2022-01-17T09:57:05.134432Z 2022-01-17T09:57:05.023792Z 2022-01-17T09:57:03.866913Z 2022-01-17T09:57:03.772326Z 2022-01-17T09:57:03.670837Z 2022-01-17T09:57:03.607591Z 2022-01-17T09:57:03.502385Z 2022-01-17T09:57:03.425051Z 2022-01-17T09:57:03.352074Z 2022-01-17T10:00:11.336594Z 2022-01-17T10:00:11.184694Z 2022-01-17T10:00:11.095926Z 2022-01-17T10:00:09.940031Z 2022-01-17T10:00:09.787739Z 2022-01-17T10:00:09.681876Z 2022-01-17T10:00:09.583513Z 2022-01-17T10:00:09.505238Z 2022-01-17T10:00:09.417094Z 2022-01-17T10:00:09.346846Z 2022-01-17T10:03:08.180605Z 2022-01-17T10:03:08.070318Z 2022-01-17T10:03:07.939372Z 2022-01-17T10:03:07.012369Z 2022-01-17T10:03:06.936192Z 2022-01-17T10:03:06.833523Z 2022-01-17T10:03:06.733872Z 2022-01-17T10:03:06.643790Z 2022-01-17T10:03:06.560965Z 2022-01-17T10:03:06.478525Z", + "started_diff": 6.607505399999999, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 14, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-17T10:08:22.867219+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-launching.json new file mode 100644 index 00000000..729c9a3d --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T09_07_03_881_0000-launching.json @@ -0,0 +1,634 @@ +{ + "ended": "2022-01-17T10:54:00.584224+00:00", + "id": "run-2022-01-17T09_07_03_881_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 18139.800000000003, + "max": 29067.933333333334, + "mean": 10069.052813852813, + "median": 4588.066666666667, + "min": 1206.6666666666665, + "non_zero_mean": 10069.052813852813, + "non_zero_median": 4588.066666666667, + "percentile25": 1770.5333333333333, + "percentile75": 19910.333333333336, + "percentile90": 23308.86666666667, + "percentile99": 27696.133333333324, + "percentile999": 28930.753333333345, + "range": 27861.266666666666, + "samples": 77, + "stdev": 9348.036358698455, + "sum": 775317.0666666667 + } + }, + "cpu": { + "system": { + "iqr": 1.428000000000001, + "max": 1.8113333333333341, + "mean": 0.6932034632034632, + "median": 0.2966666666666678, + "min": 0.02999999999999924, + "non_zero_mean": 0.6932034632034632, + "non_zero_median": 0.2966666666666678, + "percentile25": 0.03866666666666561, + "percentile75": 1.4666666666666666, + "percentile90": 1.6826666666666668, + "percentile99": 1.7763733333333345, + "percentile999": 1.8078373333333344, + "range": 1.781333333333335, + "samples": 77, + "stdev": 0.7073585824731151, + "sum": 53.37666666666669 + }, + "user": { + "iqr": 5.7806666666666535, + "max": 6.64266666666665, + "mean": 2.9251948051948053, + "median": 1.3893333333333506, + "min": 0.11266666666665515, + "non_zero_mean": 2.9251948051948053, + "non_zero_median": 1.3893333333333506, + "percentile25": 0.3139999999999873, + "percentile75": 6.094666666666641, + "percentile90": 6.313333333333336, + "percentile99": 6.548933333333333, + "percentile999": 6.633293333333319, + "range": 6.529999999999995, + "samples": 77, + "stdev": 2.7653866333696344, + "sum": 225.24000000000007 + } + }, + "entropy_available_bits": { + "iqr": 4.8, + "max": 424.06666666666666, + "mean": 23.798268398268398, + "median": 0.7333333333333334, + "min": 0.0, + "non_zero_mean": 40.72148148148148, + "non_zero_median": 4.4, + "percentile25": 0.0, + "percentile75": 4.8, + "percentile90": 5.573333333333337, + "percentile99": 423.05333333333334, + "percentile999": 423.9653333333333, + "range": 424.06666666666666, + "samples": 77, + "stdev": 93.92012842451767, + "sum": 1832.4666666666667 + }, + "memory": { + "used": { + "iqr": 5615738880.0, + "max": 17926086656.0, + "mean": 9029304639.16883, + "median": 6865965056.0, + "min": 5119324160.0, + "non_zero_mean": 9029304639.16883, + "non_zero_median": 6865965056.0, + "percentile25": 6369681408.0, + "percentile75": 11985420288.0, + "percentile90": 15800417484.8, + "percentile99": 17450301849.6, + "percentile999": 17878508175.360004, + "range": 12806762496.0, + "samples": 77, + "stdev": 3837952766.0601597, + "sum": 695256457216.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003463203463203463, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06399999999999864, + "percentile999": 0.24640000000000176, + "range": 0.26666666666666666, + "samples": 77, + "stdev": 0.030389487055903455, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 843.4666666666668, + "max": 1795.5333333333333, + "mean": 469.63982683982687, + "median": 58.19999999999999, + "min": 0.26666666666666666, + "non_zero_mean": 469.63982683982687, + "non_zero_median": 58.19999999999999, + "percentile25": 3.533333333333333, + "percentile75": 847.0000000000001, + "percentile90": 1566.2933333333333, + "percentile99": 1742.8906666666662, + "percentile999": 1790.2690666666672, + "range": 1795.2666666666667, + "samples": 77, + "stdev": 595.0521284609732, + "sum": 36162.266666666656 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2163.333333333333, + "max": 3485.0, + "mean": 1739.365367965368, + "median": 1661.5333333333333, + "min": 376.1333333333333, + "non_zero_mean": 1739.365367965368, + "non_zero_median": 1661.5333333333333, + "percentile25": 660.4, + "percentile75": 2823.733333333333, + "percentile90": 3176.4133333333334, + "percentile99": 3473.0933333333332, + "percentile999": 3483.8093333333336, + "range": 3108.866666666667, + "samples": 77, + "stdev": 1123.1547166865525, + "sum": 133931.13333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.2080000000000003, + "max": 0.38266666666666727, + "mean": 0.13794805194805196, + "median": 0.14333333333333276, + "min": 0.010666666666665492, + "non_zero_mean": 0.13794805194805196, + "non_zero_median": 0.14333333333333276, + "percentile25": 0.024666666666666497, + "percentile75": 0.2326666666666668, + "percentile90": 0.2999999999999999, + "percentile99": 0.36949333333333284, + "percentile999": 0.38134933333333393, + "range": 0.3720000000000018, + "samples": 77, + "stdev": 0.12140024982554551, + "sum": 10.621999999999998 + }, + "user": { + "iqr": 0.3006666666666661, + "max": 0.7626666666666674, + "mean": 0.20509090909090907, + "median": 0.17266666666666783, + "min": 0.012000000000000455, + "non_zero_mean": 0.20509090909090907, + "non_zero_median": 0.17266666666666783, + "percentile25": 0.039333333333331666, + "percentile75": 0.33999999999999775, + "percentile90": 0.42146666666666743, + "percentile99": 0.6851466666666655, + "percentile999": 0.754914666666668, + "range": 0.7506666666666669, + "samples": 77, + "stdev": 0.19207308273720813, + "sum": 15.792000000000003 + } + }, + "entropy_available_bits": { + "iqr": 3.3333333333333335, + "max": 213.33333333333334, + "mean": 12.077056277056277, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 21.134848484848487, + "non_zero_median": 2.666666666666667, + "percentile25": 0.0, + "percentile75": 3.3333333333333335, + "percentile90": 4.213333333333335, + "percentile99": 212.72533333333334, + "percentile999": 213.27253333333334, + "range": 213.33333333333334, + "samples": 77, + "stdev": 46.9039856221608, + "sum": 929.9333333333334 + }, + "memory": { + "used": { + "iqr": 456826880.0, + "max": 1101234176.0, + "mean": 577748194.0779221, + "median": 407105536.0, + "min": 366157824.0, + "non_zero_mean": 577748194.0779221, + "non_zero_median": 407105536.0, + "percentile25": 376143872.0, + "percentile75": 832970752.0, + "percentile90": 1006940979.2000003, + "percentile99": 1096393523.2, + "percentile999": 1100750110.72, + "range": 735076352.0, + "samples": 77, + "stdev": 259815285.80103242, + "sum": 44486610944.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1638.4, + "max": 55599162.6, + "mean": 738663.825974026, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2472918.0260869567, + "non_zero_median": 47513.6, + "percentile25": 0.0, + "percentile75": 1638.4, + "percentile90": 70997.33333333334, + "percentile99": 13488655.42933305, + "percentile999": 51388111.8829337, + "range": 55599162.6, + "samples": 77, + "stdev": 6334326.46088567, + "sum": 56877114.60000001 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 6.24418, + "mean": 0.1372326883116883, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.5561535263157895, + "non_zero_median": 0.084181, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.15392520000000015, + "percentile99": 2.1940518399999727, + "percentile999": 5.839167184000035, + "range": 6.24418, + "samples": 77, + "stdev": 0.7270617716838866, + "sum": 10.566917000000004 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.693992, + "max": 12.524207, + "mean": 0.5434054805194806, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.1308708648648649, + "non_zero_median": 0.7038, + "percentile25": 0.0, + "percentile75": 0.693992, + "percentile90": 0.8001118, + "percentile99": 7.878906119999969, + "percentile999": 12.059676912000041, + "range": 12.524207, + "samples": 77, + "stdev": 1.6277626486048802, + "sum": 41.84222199999999 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 2.933333333333333, + "mean": 0.11688311688311688, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.25, + "non_zero_median": 2.166666666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.578666666666664, + "percentile999": 2.8978666666666695, + "range": 2.933333333333333, + "samples": 77, + "stdev": 0.5146003779867202, + "sum": 9.0 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9988.266666666666, + "mean": 427.96969696969694, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8238.416666666666, + "non_zero_median": 8302.6, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9987.709333333332, + "percentile999": 9988.210933333334, + "range": 9988.266666666666, + "samples": 77, + "stdev": 1883.6511944632896, + "sum": 32953.666666666664 + }, + "pg_stat_database_blks_hit": { + "iqr": 20955.266666666666, + "max": 107452.4, + "mean": 22458.24329004329, + "median": 12070.866666666667, + "min": 1198.1333333333334, + "non_zero_mean": 22458.24329004329, + "non_zero_median": 12070.866666666667, + "percentile25": 7843.533333333334, + "percentile75": 28798.8, + "percentile90": 45320.96000000001, + "percentile99": 101859.40799999995, + "percentile999": 106893.10080000004, + "range": 106254.26666666666, + "samples": 77, + "stdev": 23003.30303808733, + "sum": 1729284.7333333339 + }, + "pg_stat_database_blks_read": { + "iqr": 3.2, + "max": 12.8, + "mean": 2.2536796536796535, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 3.692198581560284, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 3.2, + "percentile90": 6.293333333333334, + "percentile99": 12.445333333333332, + "percentile999": 12.764533333333338, + "range": 12.8, + "samples": 77, + "stdev": 3.2864783932594603, + "sum": 173.53333333333327 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 77, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.2, + "max": 12.8, + "mean": 2.2536796536796535, + "median": 0.6, + "min": 0.0, + "non_zero_mean": 3.692198581560284, + "non_zero_median": 2.4, + "percentile25": 0.0, + "percentile75": 3.2, + "percentile90": 6.293333333333334, + "percentile99": 12.445333333333332, + "percentile999": 12.764533333333338, + "range": 12.8, + "samples": 77, + "stdev": 3.2864783932594603, + "sum": 173.53333333333327 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 1.4, + "mean": 0.19307359307359306, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.991111111111111, + "non_zero_median": 1.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 1.0, + "percentile99": 1.2479999999999989, + "percentile999": 1.3848000000000011, + "range": 1.4, + "samples": 77, + "stdev": 0.4123225353625095, + "sum": 14.866666666666665 + }, + "pg_stat_database_tup_fetched": { + "iqr": 11753.733333333334, + "max": 54165.0, + "mean": 11988.181818181818, + "median": 6436.533333333334, + "min": 581.4, + "non_zero_mean": 11988.181818181818, + "non_zero_median": 6436.533333333334, + "percentile25": 4076.6666666666665, + "percentile75": 15830.4, + "percentile90": 25430.54666666667, + "percentile99": 51476.11999999998, + "percentile999": 53896.11200000002, + "range": 53583.6, + "samples": 77, + "stdev": 11903.744959252455, + "sum": 923090.0000000001 + }, + "pg_stat_database_tup_inserted": { + "iqr": 29.533333333333335, + "max": 102.0, + "mean": 17.633766233766234, + "median": 6.2, + "min": 0.0, + "non_zero_mean": 26.623529411764707, + "non_zero_median": 23.133333333333333, + "percentile25": 0.0, + "percentile75": 29.533333333333335, + "percentile90": 49.09333333333334, + "percentile99": 92.06933333333328, + "percentile999": 101.00693333333342, + "range": 102.0, + "samples": 77, + "stdev": 24.621786158020726, + "sum": 1357.8000000000004 + }, + "pg_stat_database_tup_returned": { + "iqr": 17650.199999999997, + "max": 84151.6, + "mean": 18388.605194805194, + "median": 10102.733333333334, + "min": 1573.7333333333333, + "non_zero_mean": 18388.605194805194, + "non_zero_median": 10102.733333333334, + "percentile25": 6867.866666666667, + "percentile75": 24518.066666666666, + "percentile90": 38994.38666666667, + "percentile99": 76625.11733333328, + "percentile999": 83398.95173333341, + "range": 82577.86666666667, + "samples": 77, + "stdev": 18161.502652118208, + "sum": 1415922.5999999992 + }, + "pg_stat_database_tup_updated": { + "iqr": 31.733333333333334, + "max": 82.66666666666667, + "mean": 17.805194805194805, + "median": 5.866666666666666, + "min": 0.0, + "non_zero_mean": 18.527027027027028, + "non_zero_median": 11.666666666666668, + "percentile25": 0.26666666666666666, + "percentile75": 32.0, + "percentile90": 48.70666666666668, + "percentile99": 72.12799999999993, + "percentile999": 81.61280000000009, + "range": 82.66666666666667, + "samples": 77, + "stdev": 20.63218619443524, + "sum": 1371.0000000000005 + }, + "pg_stat_database_xact_commit": { + "iqr": 297.1333333333333, + "max": 620.6, + "mean": 158.4233766233766, + "median": 65.46666666666667, + "min": 2.8666666666666667, + "non_zero_mean": 158.4233766233766, + "non_zero_median": 65.46666666666667, + "percentile25": 10.0, + "percentile75": 307.1333333333333, + "percentile90": 363.4000000000001, + "percentile99": 589.0853333333331, + "percentile999": 617.4485333333337, + "range": 617.7333333333333, + "samples": 77, + "stdev": 168.6937329251282, + "sum": 12198.600000000002 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 77, + "stdev": 0.0, + "sum": 5.133333333333335 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003463203463203463, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06399999999999864, + "percentile999": 0.24640000000000176, + "range": 0.26666666666666666, + "samples": 77, + "stdev": 0.030389487055903455, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 99.13333333333334, + "max": 150.46666666666667, + "mean": 58.26493506493507, + "median": 39.733333333333334, + "min": 0.5333333333333333, + "non_zero_mean": 58.26493506493507, + "non_zero_median": 39.733333333333334, + "percentile25": 4.2, + "percentile75": 103.33333333333334, + "percentile90": 130.53333333333333, + "percentile99": 148.94666666666666, + "percentile999": 150.31466666666668, + "range": 149.93333333333334, + "samples": 77, + "stdev": 51.69522128199908, + "sum": 4486.4 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "60,63,70,61,67,65,66,75,79,62,71,82,73,81,74,72,80,69,86,83,84,77,87,90,78,85,88,89,91,94,95,99,98,96,100,93,101,103,104,106,107,110,109,111,105,112,114,115,118,119,120,116,121,122,124,113,125,127,130,126,128,123,131,133,132,135,117,136,129,137,138,139,140,134,143,141,146,149,153,145,150,142,154,157,155,159,158,162,156,160,163,165,164,167,161,166,170,169,168,171,172,176,173,180,174,175,184,183,179,177,181,178,186,187,195,192,182,194,190,185,197,188,189,191,193,196,198,200,199,203,202,201,204,207,205,206,210,209,208,211,212,217,214,221,220,213,216,223,219,218,225,224,226,222,233,227,230,231,228,229,232,235,239,215,238,237,236,241,244,247,242,246,245,240,243,253,249,252,251,257,248,255,263,260,256,234,261,264,254,262,250,259,258,265,269,266,267,268,270,271,276,277,273,272,280,275,278,279,286,274,288,285,287,283,291,282,284,292,293,297,289,296,295,294,305,306,290,281,299,302,301,300,311,309,310,304,312,298,307,308,303,313,317,314,321,319,315,318,316,320,324,332,323,327,325,334,328,322,330,336,329,339,338,331,337,335,326,347,341,342,349,350,345,333,348,354,357,346,358,356,351,340,344,353,352,359,355,360,364,362,343,365,363,366,368,370,361,367,371,369,373,378,376,372,375,385,381,386,377,384,383,380,379,382,374,394,388,387,389,395,391,396,393,392,390,400,402,397,399,406,398,401,403,412,405,408,410,407,404,411,415,413,409,416,421,420,414,422,417,419,418,427,424,428,431,430,429,425,433,423,435,434,426,432,443,441,439,447,436,445,437,452,438,444,446,450,456,455,449,442,454,457,451,459,448,440,458,461,464,460,453,465,462,466,468,467,463,471,469,470,474,475,472,477,473,479,476,478,480,488,484,494,483,481,489,485,487,482,493,492,495,486,497,491,499,490,496,500,498,502,501,504,503,509,505,508,506,507,511,512,510,515,514,513,520,518,516,517,519,522,521,524,523,525,528,530,526,527,529,531,533,541,532,535,538,536,546,537,539,543,540,544,542,545,550,547,554,553,552,549,555,558,560,559,557,534,561,548,562,556,563,566,564,565,567,551,568,569,570,571", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-17T10:10:13.513221Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "FAIL", + "results": { + "created": "2022-01-17T10:10:19.061084Z 2022-01-17T10:10:18.998466Z 2022-01-17T10:10:18.676439Z 2022-01-17T10:10:18.643516Z 2022-01-17T10:10:18.626523Z 2022-01-17T10:10:18.616404Z 2022-01-17T10:10:18.019722Z 2022-01-17T10:10:17.993517Z 2022-01-17T10:10:17.984771Z 2022-01-17T10:10:17.858985Z 2022-01-17T10:10:17.748992Z 2022-01-17T10:10:17.724740Z 2022-01-17T10:10:17.498678Z 2022-01-17T10:10:16.763252Z 2022-01-17T10:10:16.648217Z 2022-01-17T10:10:16.546095Z 2022-01-17T10:10:16.535930Z 2022-01-17T10:10:16.526178Z 2022-01-17T10:10:16.484935Z 2022-01-17T10:10:16.469705Z 2022-01-17T10:10:16.396173Z 2022-01-17T10:10:15.791187Z 2022-01-17T10:10:15.731478Z 2022-01-17T10:10:15.641704Z 2022-01-17T10:10:15.626222Z 2022-01-17T10:10:15.588056Z 2022-01-17T10:10:15.580101Z 2022-01-17T10:10:15.296824Z 2022-01-17T10:10:14.737111Z 2022-01-17T10:10:14.645489Z 2022-01-17T10:10:14.552880Z 2022-01-17T10:10:14.351781Z 2022-01-17T10:10:14.142457Z 2022-01-17T10:10:13.751970Z 2022-01-17T10:10:13.513221Z 2022-01-17T10:10:38.493783Z 2022-01-17T10:10:37.850053Z 2022-01-17T10:10:36.579652Z 2022-01-17T10:10:35.977357Z 2022-01-17T10:10:34.926401Z 2022-01-17T10:10:33.840450Z 2022-01-17T10:10:33.098436Z 2022-01-17T10:10:32.686664Z 2022-01-17T10:10:32.047803Z 2022-01-17T10:10:31.803540Z 2022-01-17T10:10:31.087562Z 2022-01-17T10:10:30.933218Z 2022-01-17T10:10:30.324245Z 2022-01-17T10:10:29.974223Z 2022-01-17T10:10:27.711367Z 2022-01-17T10:10:27.028271Z 2022-01-17T10:10:26.750972Z 2022-01-17T10:10:26.638169Z 2022-01-17T10:10:26.314464Z 2022-01-17T10:10:25.935312Z 2022-01-17T10:10:25.833284Z 2022-01-17T10:10:25.449271Z 2022-01-17T10:10:25.431628Z 2022-01-17T10:10:25.209695Z 2022-01-17T10:10:24.963331Z 2022-01-17T10:10:24.907690Z 2022-01-17T10:10:24.728393Z 2022-01-17T10:10:24.578546Z 2022-01-17T10:10:24.327456Z 2022-01-17T10:10:24.245624Z 2022-01-17T10:10:24.011184Z 2022-01-17T10:10:23.839770Z 2022-01-17T10:10:23.818936Z 2022-01-17T10:10:23.719850Z 2022-01-17T10:10:23.713896Z 2022-01-17T10:10:23.669712Z 2022-01-17T10:10:23.454797Z 2022-01-17T10:10:23.356395Z 2022-01-17T10:10:23.038760Z 2022-01-17T10:10:22.983329Z 2022-01-17T10:10:22.556186Z 2022-01-17T10:10:22.548202Z 2022-01-17T10:10:22.231927Z 2022-01-17T10:10:22.134081Z 2022-01-17T10:10:22.079071Z 2022-01-17T10:10:22.002761Z 2022-01-17T10:10:21.928943Z 2022-01-17T10:10:21.667560Z 2022-01-17T10:10:21.477538Z 2022-01-17T10:10:21.447011Z 2022-01-17T10:10:21.387870Z 2022-01-17T10:10:21.281732Z 2022-01-17T10:10:21.169123Z 2022-01-17T10:10:21.026915Z 2022-01-17T10:10:20.919680Z 2022-01-17T10:10:20.847832Z 2022-01-17T10:10:20.803686Z 2022-01-17T10:10:20.606887Z 2022-01-17T10:10:20.419790Z 2022-01-17T10:10:20.296922Z 2022-01-17T10:10:20.130970Z 2022-01-17T10:10:20.011064Z 2022-01-17T10:10:19.797042Z 2022-01-17T10:10:19.318436Z 2022-01-17T10:10:19.248770Z 2022-01-17T10:15:16.778901Z 2022-01-17T10:15:16.552087Z 2022-01-17T10:15:15.394658Z 2022-01-17T10:15:15.343120Z 2022-01-17T10:15:15.322301Z 2022-01-17T10:15:15.016882Z 2022-01-17T10:15:14.548726Z 2022-01-17T10:15:13.396647Z 2022-01-17T10:15:13.341520Z 2022-01-17T10:15:12.993468Z 2022-01-17T10:15:12.952694Z 2022-01-17T10:15:12.819427Z 2022-01-17T10:15:12.655955Z 2022-01-17T10:15:11.946554Z 2022-01-17T10:15:11.894676Z 2022-01-17T10:15:11.834399Z 2022-01-17T10:15:11.694943Z 2022-01-17T10:15:11.582581Z 2022-01-17T10:15:11.356300Z 2022-01-17T10:15:11.262827Z 2022-01-17T10:15:11.094458Z 2022-01-17T10:15:10.559467Z 2022-01-17T10:15:10.403540Z 2022-01-17T10:15:09.920050Z 2022-01-17T10:15:09.707678Z 2022-01-17T10:15:09.628680Z 2022-01-17T10:15:09.475480Z 2022-01-17T10:15:09.473015Z 2022-01-17T10:15:09.232604Z 2022-01-17T10:15:08.890781Z 2022-01-17T10:15:08.846757Z 2022-01-17T10:15:08.836274Z 2022-01-17T10:15:08.791507Z 2022-01-17T10:15:08.609092Z 2022-01-17T10:15:08.516565Z 2022-01-17T10:15:08.158624Z 2022-01-17T10:15:08.110934Z 2022-01-17T10:15:07.985111Z 2022-01-17T10:15:07.454415Z 2022-01-17T10:15:07.420025Z 2022-01-17T10:15:07.348909Z 2022-01-17T10:15:07.207582Z 2022-01-17T10:15:07.103117Z 2022-01-17T10:15:07.026840Z 2022-01-17T10:15:06.697654Z 2022-01-17T10:15:06.386468Z 2022-01-17T10:15:06.375103Z 2022-01-17T10:15:05.905536Z 2022-01-17T10:15:05.876033Z 2022-01-17T10:15:05.739891Z 2022-01-17T10:15:05.250373Z 2022-01-17T10:15:05.219562Z 2022-01-17T10:15:05.210962Z 2022-01-17T10:15:05.196155Z 2022-01-17T10:15:05.085075Z 2022-01-17T10:15:05.037247Z 2022-01-17T10:15:04.979357Z 2022-01-17T10:15:04.867560Z 2022-01-17T10:15:04.851967Z 2022-01-17T10:15:04.771281Z 2022-01-17T10:15:04.688522Z 2022-01-17T10:15:04.083882Z 2022-01-17T10:15:04.055967Z 2022-01-17T10:15:03.984592Z 2022-01-17T10:15:03.928892Z 2022-01-17T10:15:03.685697Z 2022-01-17T10:15:03.567703Z 2022-01-17T10:15:03.512039Z 2022-01-17T10:15:03.335799Z 2022-01-17T10:15:03.271285Z 2022-01-17T10:15:03.195245Z 2022-01-17T10:15:02.651382Z 2022-01-17T10:15:02.632253Z 2022-01-17T10:15:02.465408Z 2022-01-17T10:15:02.107683Z 2022-01-17T10:15:02.048209Z 2022-01-17T10:15:02.007495Z 2022-01-17T10:15:01.941960Z 2022-01-17T10:15:01.894211Z 2022-01-17T10:15:01.695556Z 2022-01-17T10:15:01.601902Z 2022-01-17T10:15:01.544615Z 2022-01-17T10:15:01.490347Z 2022-01-17T10:15:01.266323Z 2022-01-17T10:15:01.104366Z 2022-01-17T10:15:01.086655Z 2022-01-17T10:15:00.954810Z 2022-01-17T10:15:00.941279Z 2022-01-17T10:15:00.938328Z 2022-01-17T10:15:00.610202Z 2022-01-17T10:15:00.533288Z 2022-01-17T10:15:00.523125Z 2022-01-17T10:15:00.185784Z 2022-01-17T10:15:00.158189Z 2022-01-17T10:15:00.135992Z 2022-01-17T10:15:00.046020Z 2022-01-17T10:14:59.929447Z 2022-01-17T10:14:59.925557Z 2022-01-17T10:14:59.430230Z 2022-01-17T10:14:59.120000Z 2022-01-17T10:19:22.546008Z 2022-01-17T10:19:22.512372Z 2022-01-17T10:19:22.266997Z 2022-01-17T10:19:22.201587Z 2022-01-17T10:19:22.134056Z 2022-01-17T10:19:22.062302Z 2022-01-17T10:19:21.857164Z 2022-01-17T10:19:21.622126Z 2022-01-17T10:19:21.556292Z 2022-01-17T10:19:20.956650Z 2022-01-17T10:19:20.952377Z 2022-01-17T10:19:19.945026Z 2022-01-17T10:19:19.666166Z 2022-01-17T10:19:19.644066Z 2022-01-17T10:19:19.541239Z 2022-01-17T10:19:19.373505Z 2022-01-17T10:19:19.091313Z 2022-01-17T10:19:19.059344Z 2022-01-17T10:19:19.037846Z 2022-01-17T10:19:18.986310Z 2022-01-17T10:19:18.969257Z 2022-01-17T10:19:18.807428Z 2022-01-17T10:19:18.637311Z 2022-01-17T10:19:18.492311Z 2022-01-17T10:19:18.109705Z 2022-01-17T10:19:18.025713Z 2022-01-17T10:19:18.024700Z 2022-01-17T10:19:17.915231Z 2022-01-17T10:19:17.503725Z 2022-01-17T10:19:17.472912Z 2022-01-17T10:19:17.443254Z 2022-01-17T10:19:17.099174Z 2022-01-17T10:19:17.070402Z 2022-01-17T10:19:16.848639Z 2022-01-17T10:19:16.784962Z 2022-01-17T10:19:16.673472Z 2022-01-17T10:19:16.550196Z 2022-01-17T10:19:16.522888Z 2022-01-17T10:19:16.473280Z 2022-01-17T10:19:16.390293Z 2022-01-17T10:19:16.111823Z 2022-01-17T10:19:16.080308Z 2022-01-17T10:19:16.026445Z 2022-01-17T10:19:16.021397Z 2022-01-17T10:19:16.009170Z 2022-01-17T10:19:15.982523Z 2022-01-17T10:19:15.768684Z 2022-01-17T10:19:15.759309Z 2022-01-17T10:19:15.732399Z 2022-01-17T10:19:15.542004Z 2022-01-17T10:19:15.170196Z 2022-01-17T10:19:15.069070Z 2022-01-17T10:19:15.046374Z 2022-01-17T10:19:14.933756Z 2022-01-17T10:19:14.630700Z 2022-01-17T10:19:14.598367Z 2022-01-17T10:19:14.572475Z 2022-01-17T10:19:14.474675Z 2022-01-17T10:19:14.383828Z 2022-01-17T10:19:14.374461Z 2022-01-17T10:19:14.115007Z 2022-01-17T10:19:14.043395Z 2022-01-17T10:19:13.935704Z 2022-01-17T10:19:13.862971Z 2022-01-17T10:19:13.738899Z 2022-01-17T10:19:13.708433Z 2022-01-17T10:19:13.688947Z 2022-01-17T10:19:13.600144Z 2022-01-17T10:19:13.542938Z 2022-01-17T10:19:13.520646Z 2022-01-17T10:19:13.481982Z 2022-01-17T10:19:13.417549Z 2022-01-17T10:19:13.249079Z 2022-01-17T10:19:13.208404Z 2022-01-17T10:19:13.002750Z 2022-01-17T10:19:12.963569Z 2022-01-17T10:19:12.954211Z 2022-01-17T10:19:12.949320Z 2022-01-17T10:19:12.620298Z 2022-01-17T10:19:12.550464Z 2022-01-17T10:19:12.541283Z 2022-01-17T10:19:12.494964Z 2022-01-17T10:19:12.447818Z 2022-01-17T10:19:12.388745Z 2022-01-17T10:19:12.282202Z 2022-01-17T10:19:12.275721Z 2022-01-17T10:19:12.263471Z 2022-01-17T10:19:12.237322Z 2022-01-17T10:19:12.135509Z 2022-01-17T10:19:12.076655Z 2022-01-17T10:19:12.013198Z 2022-01-17T10:19:11.855862Z 2022-01-17T10:19:11.845928Z 2022-01-17T10:19:11.550969Z 2022-01-17T10:19:11.545212Z 2022-01-17T10:19:11.440839Z 2022-01-17T10:19:11.333612Z 2022-01-17T10:19:11.231314Z 2022-01-17T10:19:11.228868Z 2022-01-17T10:19:11.126374Z 2022-01-17T10:23:19.373516Z 2022-01-17T10:23:19.044829Z 2022-01-17T10:23:18.950456Z 2022-01-17T10:23:18.602339Z 2022-01-17T10:23:18.543001Z 2022-01-17T10:23:18.191653Z 2022-01-17T10:23:18.149422Z 2022-01-17T10:23:18.036939Z 2022-01-17T10:23:17.828045Z 2022-01-17T10:23:17.586362Z 2022-01-17T10:23:17.455411Z 2022-01-17T10:23:17.338387Z 2022-01-17T10:23:17.295040Z 2022-01-17T10:23:17.025183Z 2022-01-17T10:23:17.008578Z 2022-01-17T10:23:16.586523Z 2022-01-17T10:23:16.431681Z 2022-01-17T10:23:16.393197Z 2022-01-17T10:23:16.353270Z 2022-01-17T10:23:16.301176Z 2022-01-17T10:23:16.287825Z 2022-01-17T10:23:16.264509Z 2022-01-17T10:23:16.251641Z 2022-01-17T10:23:15.550162Z 2022-01-17T10:23:15.469691Z 2022-01-17T10:23:15.363697Z 2022-01-17T10:23:15.244364Z 2022-01-17T10:23:15.073101Z 2022-01-17T10:23:15.038938Z 2022-01-17T10:23:14.926674Z 2022-01-17T10:23:14.962352Z 2022-01-17T10:23:14.872417Z 2022-01-17T10:23:14.715650Z 2022-01-17T10:23:14.707324Z 2022-01-17T10:23:14.661671Z 2022-01-17T10:23:14.627443Z 2022-01-17T10:23:14.615401Z 2022-01-17T10:23:14.565308Z 2022-01-17T10:23:14.553439Z 2022-01-17T10:23:14.403078Z 2022-01-17T10:23:14.221519Z 2022-01-17T10:23:14.183090Z 2022-01-17T10:23:14.095244Z 2022-01-17T10:23:14.043695Z 2022-01-17T10:23:13.924184Z 2022-01-17T10:23:13.836340Z 2022-01-17T10:23:13.710561Z 2022-01-17T10:23:13.361471Z 2022-01-17T10:23:13.321793Z 2022-01-17T10:23:13.282639Z 2022-01-17T10:23:13.095284Z 2022-01-17T10:23:12.917657Z 2022-01-17T10:23:12.872353Z 2022-01-17T10:23:12.816763Z 2022-01-17T10:23:12.760347Z 2022-01-17T10:23:12.718927Z 2022-01-17T10:23:12.433014Z 2022-01-17T10:23:12.412796Z 2022-01-17T10:23:12.345860Z 2022-01-17T10:23:12.262039Z 2022-01-17T10:23:12.256216Z 2022-01-17T10:23:12.248668Z 2022-01-17T10:23:12.230375Z 2022-01-17T10:23:12.234371Z 2022-01-17T10:23:12.024696Z 2022-01-17T10:23:11.876269Z 2022-01-17T10:23:11.844108Z 2022-01-17T10:23:11.842415Z 2022-01-17T10:23:11.678113Z 2022-01-17T10:23:11.569600Z 2022-01-17T10:23:11.504958Z 2022-01-17T10:23:11.387555Z 2022-01-17T10:23:11.335407Z 2022-01-17T10:23:11.254130Z 2022-01-17T10:23:10.871502Z 2022-01-17T10:23:10.824643Z 2022-01-17T10:23:10.789499Z 2022-01-17T10:23:10.756350Z 2022-01-17T10:23:10.700461Z 2022-01-17T10:23:10.672067Z 2022-01-17T10:23:10.622675Z 2022-01-17T10:23:10.472680Z 2022-01-17T10:23:10.452540Z 2022-01-17T10:23:10.193996Z 2022-01-17T10:23:10.193706Z 2022-01-17T10:23:10.179258Z 2022-01-17T10:23:10.066825Z 2022-01-17T10:23:09.921658Z 2022-01-17T10:23:09.882933Z 2022-01-17T10:23:09.880167Z 2022-01-17T10:23:09.667635Z 2022-01-17T10:23:09.644714Z 2022-01-17T10:23:09.621508Z 2022-01-17T10:23:09.573984Z 2022-01-17T10:23:09.561662Z 2022-01-17T10:23:09.517474Z 2022-01-17T10:23:09.466436Z 2022-01-17T10:23:09.237262Z 2022-01-17T10:23:09.212249Z 2022-01-17T10:23:08.824093Z 2022-01-17T10:27:37.414072Z 2022-01-17T10:27:36.963506Z 2022-01-17T10:27:36.845941Z 2022-01-17T10:27:36.834687Z 2022-01-17T10:27:35.418538Z 2022-01-17T10:27:35.270673Z 2022-01-17T10:27:35.237714Z 2022-01-17T10:27:35.148584Z 2022-01-17T10:27:34.938160Z 2022-01-17T10:27:34.468111Z 2022-01-17T10:27:34.438342Z 2022-01-17T10:27:34.211088Z 2022-01-17T10:27:34.092754Z 2022-01-17T10:27:33.931744Z 2022-01-17T10:27:33.922230Z 2022-01-17T10:27:33.911848Z 2022-01-17T10:27:33.659130Z 2022-01-17T10:27:33.584567Z 2022-01-17T10:27:33.312655Z 2022-01-17T10:27:33.248894Z 2022-01-17T10:27:33.171963Z 2022-01-17T10:27:32.981192Z 2022-01-17T10:27:32.993420Z 2022-01-17T10:27:32.735626Z 2022-01-17T10:27:32.533166Z 2022-01-17T10:27:32.482457Z 2022-01-17T10:27:31.882001Z 2022-01-17T10:27:31.841158Z 2022-01-17T10:27:31.778695Z 2022-01-17T10:27:31.610362Z 2022-01-17T10:27:31.598955Z 2022-01-17T10:27:31.588706Z 2022-01-17T10:27:31.489409Z 2022-01-17T10:27:31.365953Z 2022-01-17T10:27:31.269046Z 2022-01-17T10:27:31.149716Z 2022-01-17T10:27:30.937862Z 2022-01-17T10:27:30.789328Z 2022-01-17T10:27:30.498595Z 2022-01-17T10:27:30.389705Z 2022-01-17T10:27:30.300308Z 2022-01-17T10:27:30.291053Z 2022-01-17T10:27:30.269316Z 2022-01-17T10:27:30.196960Z 2022-01-17T10:27:30.017935Z 2022-01-17T10:27:30.003567Z 2022-01-17T10:27:29.999953Z 2022-01-17T10:27:29.898380Z 2022-01-17T10:27:29.897290Z 2022-01-17T10:27:29.849802Z 2022-01-17T10:27:29.576582Z 2022-01-17T10:27:29.398963Z 2022-01-17T10:27:29.256068Z 2022-01-17T10:27:29.274665Z 2022-01-17T10:27:29.202624Z 2022-01-17T10:27:28.778792Z 2022-01-17T10:27:28.732361Z 2022-01-17T10:27:28.619873Z 2022-01-17T10:27:28.610926Z 2022-01-17T10:27:28.577598Z 2022-01-17T10:27:28.339911Z 2022-01-17T10:27:28.303586Z 2022-01-17T10:27:28.054475Z 2022-01-17T10:27:27.759038Z 2022-01-17T10:27:27.741515Z 2022-01-17T10:27:27.627036Z 2022-01-17T10:27:27.610869Z 2022-01-17T10:27:27.607273Z 2022-01-17T10:27:27.354043Z 2022-01-17T10:27:27.297775Z 2022-01-17T10:27:27.307405Z 2022-01-17T10:27:27.213800Z 2022-01-17T10:27:27.212902Z 2022-01-17T10:27:27.180827Z 2022-01-17T10:27:27.178504Z 2022-01-17T10:27:27.125091Z 2022-01-17T10:27:27.075905Z 2022-01-17T10:27:27.053384Z 2022-01-17T10:27:26.969327Z 2022-01-17T10:27:26.872449Z 2022-01-17T10:27:26.860963Z 2022-01-17T10:27:26.745807Z 2022-01-17T10:27:26.694641Z 2022-01-17T10:27:26.465781Z 2022-01-17T10:27:26.372533Z 2022-01-17T10:27:26.297242Z 2022-01-17T10:27:26.281469Z 2022-01-17T10:27:26.197922Z 2022-01-17T10:27:26.108473Z 2022-01-17T10:27:26.083281Z 2022-01-17T10:27:25.949305Z 2022-01-17T10:27:25.837569Z 2022-01-17T10:27:25.812345Z 2022-01-17T10:27:25.591012Z 2022-01-17T10:27:25.580928Z 2022-01-17T10:27:25.373985Z 2022-01-17T10:27:25.112259Z 2022-01-17T10:27:25.027809Z 2022-01-17T10:27:25.004990Z 2022-01-17T10:27:24.922530Z", + "created_diff": 15.4200124, + "ended": "2022-01-17T10:29:13.163399Z", + "error_job_ids": " []", + "event_first": "2022-01-17T10:11:11.725744Z 2022-01-17T10:10:37.998093Z 2022-01-17T10:11:25.076687Z 2022-01-17T10:10:37.693376Z 2022-01-17T10:11:15.999665Z 2022-01-17T10:10:34.921609Z 2022-01-17T10:11:23.350536Z 2022-01-17T10:10:36.473136Z 2022-01-17T10:11:17.341730Z 2022-01-17T10:10:36.561898Z 2022-01-17T10:11:07.026751Z 2022-01-17T10:10:43.335794Z 2022-01-17T10:11:00.947642Z 2022-01-17T10:13:03.223284Z 2022-01-17T10:11:16.199172Z 2022-01-17T10:11:15.990019Z 2022-01-17T10:11:08.076998Z 2022-01-17T10:12:55.095156Z 2022-01-17T10:11:01.839041Z 2022-01-17T10:11:59.197767Z 2022-01-17T10:10:58.450352Z 2022-01-17T10:12:17.718201Z 2022-01-17T10:10:40.375542Z 2022-01-17T10:12:27.111997Z 2022-01-17T10:10:45.154624Z 2022-01-17T10:12:08.566025Z 2022-01-17T10:10:42.776650Z 2022-01-17T10:12:36.952916Z 2022-01-17T10:10:36.423448Z 2022-01-17T10:12:46.349442Z 2022-01-17T10:10:35.374551Z 2022-01-17T10:11:50.879289Z 2022-01-17T10:10:31.080381Z 2022-01-17T10:10:35.689760Z 2022-01-17T10:10:23.615113Z 2022-01-17T10:11:55.405523Z 2022-01-17T10:11:45.029372Z 2022-01-17T10:11:52.052008Z 2022-01-17T10:11:46.236286Z 2022-01-17T10:11:55.001538Z 2022-01-17T10:11:43.956898Z 2022-01-17T10:11:56.391715Z 2022-01-17T10:11:45.237027Z 2022-01-17T10:11:58.737358Z 2022-01-17T10:11:44.472918Z 2022-01-17T10:11:54.671546Z 2022-01-17T10:11:46.661571Z 2022-01-17T10:11:47.830919Z 2022-01-17T10:11:43.180768Z 2022-01-17T10:11:56.863356Z 2022-01-17T10:11:46.996812Z 2022-01-17T10:11:42.401022Z 2022-01-17T10:11:53.852679Z 2022-01-17T10:11:40.704778Z 2022-01-17T10:11:54.353650Z 2022-01-17T10:11:40.450181Z 2022-01-17T10:11:49.983449Z 2022-01-17T10:11:37.740094Z 2022-01-17T10:11:54.301521Z 2022-01-17T10:11:38.246919Z 2022-01-17T10:11:51.882515Z 2022-01-17T10:11:37.797267Z 2022-01-17T10:11:53.479713Z 2022-01-17T10:11:37.495269Z 2022-01-17T10:11:48.702828Z 2022-01-17T10:11:35.353001Z 2022-01-17T10:11:50.786737Z 2022-01-17T10:11:35.763032Z 2022-01-17T10:11:52.273980Z 2022-01-17T10:11:31.882616Z 2022-01-17T10:11:52.153108Z 2022-01-17T10:11:37.242070Z 2022-01-17T10:11:44.270557Z 2022-01-17T10:11:31.676523Z 2022-01-17T10:11:48.992987Z 2022-01-17T10:11:25.418944Z 2022-01-17T10:11:43.926533Z 2022-01-17T10:11:22.247126Z 2022-01-17T10:11:42.801601Z 2022-01-17T10:11:23.125218Z 2022-01-17T10:11:39.993721Z 2022-01-17T10:11:24.682270Z 2022-01-17T10:11:42.518610Z 2022-01-17T10:11:27.888004Z 2022-01-17T10:11:40.974744Z 2022-01-17T10:11:23.671962Z 2022-01-17T10:11:44.190429Z 2022-01-17T10:11:22.640724Z 2022-01-17T10:11:31.812659Z 2022-01-17T10:11:15.503959Z 2022-01-17T10:11:40.818460Z 2022-01-17T10:11:04.951600Z 2022-01-17T10:11:37.184660Z 2022-01-17T10:11:03.299579Z 2022-01-17T10:11:42.294598Z 2022-01-17T10:10:55.227996Z 2022-01-17T10:11:34.288222Z 2022-01-17T10:10:51.180261Z 2022-01-17T10:11:17.574631Z 2022-01-17T10:10:37.589094Z 2022-01-17T10:16:36.394054Z 2022-01-17T10:16:41.457656Z 2022-01-17T10:16:38.685745Z 2022-01-17T10:16:40.698810Z 2022-01-17T10:16:36.394054Z 2022-01-17T10:16:41.608640Z 2022-01-17T10:16:37.480609Z 2022-01-17T10:16:43.333667Z 2022-01-17T10:16:38.245132Z 2022-01-17T10:16:41.608640Z 2022-01-17T10:16:35.650381Z 2022-01-17T10:16:38.887760Z 2022-01-17T10:16:37.228843Z 2022-01-17T10:16:40.964444Z 2022-01-17T10:16:37.228843Z 2022-01-17T10:16:40.767743Z 2022-01-17T10:16:34.885769Z 2022-01-17T10:16:37.980247Z 2022-01-17T10:16:34.719996Z 2022-01-17T10:16:38.438683Z 2022-01-17T10:16:35.394019Z 2022-01-17T10:16:38.138495Z 2022-01-17T10:16:36.394054Z 2022-01-17T10:16:33.125420Z 2022-01-17T10:16:24.384160Z 2022-01-17T10:16:30.562547Z 2022-01-17T10:16:28.337054Z 2022-01-17T10:16:34.587885Z 2022-01-17T10:16:28.337054Z 2022-01-17T10:16:34.873380Z 2022-01-17T10:16:24.002316Z 2022-01-17T10:16:30.053591Z 2022-01-17T10:16:27.100113Z 2022-01-17T10:16:35.120828Z 2022-01-17T10:16:24.995906Z 2022-01-17T10:16:28.248830Z 2022-01-17T10:16:26.819342Z 2022-01-17T10:16:25.530757Z 2022-01-17T10:16:22.716730Z 2022-01-17T10:16:29.663659Z 2022-01-17T10:16:17.174928Z 2022-01-17T10:16:28.802421Z 2022-01-17T10:16:28.081569Z 2022-01-17T10:16:23.074995Z 2022-01-17T10:16:22.248279Z 2022-01-17T10:16:26.805319Z 2022-01-17T10:16:14.797077Z 2022-01-17T10:16:17.230600Z 2022-01-17T10:16:16.676731Z 2022-01-17T10:16:25.524122Z 2022-01-17T10:16:21.020280Z 2022-01-17T10:16:27.882055Z 2022-01-17T10:16:09.003010Z 2022-01-17T10:16:27.284600Z 2022-01-17T10:16:14.115267Z 2022-01-17T10:16:22.419420Z 2022-01-17T10:16:19.855709Z 2022-01-17T10:16:16.382972Z 2022-01-17T10:16:14.050188Z 2022-01-17T10:16:23.074995Z 2022-01-17T10:16:04.680999Z 2022-01-17T10:16:20.407612Z 2022-01-17T10:16:14.798426Z 2022-01-17T10:16:25.708195Z 2022-01-17T10:15:59.518968Z 2022-01-17T10:16:12.462745Z 2022-01-17T10:16:02.829169Z 2022-01-17T10:16:13.204630Z 2022-01-17T10:16:04.250032Z 2022-01-17T10:16:09.588887Z 2022-01-17T10:16:04.940077Z 2022-01-17T10:16:16.372184Z 2022-01-17T10:15:55.037052Z 2022-01-17T10:16:07.597600Z 2022-01-17T10:15:46.905882Z 2022-01-17T10:15:58.130520Z 2022-01-17T10:15:59.264113Z 2022-01-17T10:15:58.963246Z 2022-01-17T10:15:51.730341Z 2022-01-17T10:15:54.286487Z 2022-01-17T10:15:38.757324Z 2022-01-17T10:16:02.171794Z 2022-01-17T10:15:36.424638Z 2022-01-17T10:15:47.106803Z 2022-01-17T10:15:22.352722Z 2022-01-17T10:15:24.267027Z 2022-01-17T10:15:21.528484Z 2022-01-17T10:15:24.774234Z 2022-01-17T10:15:20.537303Z 2022-01-17T10:15:20.936099Z 2022-01-17T10:15:19.479362Z 2022-01-17T10:15:21.434263Z 2022-01-17T10:15:18.233448Z 2022-01-17T10:15:19.219410Z 2022-01-17T10:15:12.429266Z 2022-01-17T10:15:13.163669Z 2022-01-17T10:15:11.286331Z 2022-01-17T10:15:10.941712Z 2022-01-17T10:15:09.745184Z 2022-01-17T10:15:08.101432Z 2022-01-17T10:20:49.728001Z 2022-01-17T10:20:52.902015Z 2022-01-17T10:20:44.025110Z 2022-01-17T10:20:50.725783Z 2022-01-17T10:20:48.731622Z 2022-01-17T10:20:52.902015Z 2022-01-17T10:20:47.857934Z 2022-01-17T10:20:45.137068Z 2022-01-17T10:20:46.152404Z 2022-01-17T10:20:51.773394Z 2022-01-17T10:20:46.616080Z 2022-01-17T10:20:46.446518Z 2022-01-17T10:20:49.136250Z 2022-01-17T10:20:51.954494Z 2022-01-17T10:20:41.243256Z 2022-01-17T10:20:44.273488Z 2022-01-17T10:20:45.937726Z 2022-01-17T10:20:47.401114Z 2022-01-17T10:20:44.671919Z 2022-01-17T10:20:43.527602Z 2022-01-17T10:20:48.468772Z 2022-01-17T10:20:39.513775Z 2022-01-17T10:20:49.327627Z 2022-01-17T10:20:36.335242Z 2022-01-17T10:20:46.202926Z 2022-01-17T10:20:41.825055Z 2022-01-17T10:20:44.876518Z 2022-01-17T10:20:42.001509Z 2022-01-17T10:20:44.018332Z 2022-01-17T10:20:48.468772Z 2022-01-17T10:20:44.933626Z 2022-01-17T10:20:44.127643Z 2022-01-17T10:20:37.872895Z 2022-01-17T10:20:40.668063Z 2022-01-17T10:20:37.985889Z 2022-01-17T10:20:44.410605Z 2022-01-17T10:20:36.475884Z 2022-01-17T10:20:41.688940Z 2022-01-17T10:20:40.990394Z 2022-01-17T10:20:42.603094Z 2022-01-17T10:20:36.745703Z 2022-01-17T10:20:42.940790Z 2022-01-17T10:20:41.249064Z 2022-01-17T10:20:37.401594Z 2022-01-17T10:20:40.238098Z 2022-01-17T10:20:41.259225Z 2022-01-17T10:20:32.029240Z 2022-01-17T10:20:35.302666Z 2022-01-17T10:20:39.501058Z 2022-01-17T10:20:40.421658Z 2022-01-17T10:20:34.943378Z 2022-01-17T10:20:42.858605Z 2022-01-17T10:20:28.105878Z 2022-01-17T10:20:37.641758Z 2022-01-17T10:20:28.393983Z 2022-01-17T10:20:32.041031Z 2022-01-17T10:20:32.668206Z 2022-01-17T10:20:31.700849Z 2022-01-17T10:20:28.559201Z 2022-01-17T10:20:33.251955Z 2022-01-17T10:20:18.195402Z 2022-01-17T10:20:29.454590Z 2022-01-17T10:20:16.078067Z 2022-01-17T10:20:36.310010Z 2022-01-17T10:20:17.398924Z 2022-01-17T10:20:24.867581Z 2022-01-17T10:20:21.246366Z 2022-01-17T10:20:30.814595Z 2022-01-17T10:20:17.435670Z 2022-01-17T10:20:21.885595Z 2022-01-17T10:20:06.730469Z 2022-01-17T10:20:21.895724Z 2022-01-17T10:20:09.526832Z 2022-01-17T10:20:13.873774Z 2022-01-17T10:20:05.470020Z 2022-01-17T10:20:12.837926Z 2022-01-17T10:20:08.744142Z 2022-01-17T10:20:19.791674Z 2022-01-17T10:20:02.020735Z 2022-01-17T10:20:16.406542Z 2022-01-17T10:20:00.418895Z 2022-01-17T10:20:11.940908Z 2022-01-17T10:19:43.600882Z 2022-01-17T10:20:08.272642Z 2022-01-17T10:19:29.214024Z 2022-01-17T10:19:43.445159Z 2022-01-17T10:20:02.253751Z 2022-01-17T10:19:32.985485Z 2022-01-17T10:19:29.529006Z 2022-01-17T10:19:33.903602Z 2022-01-17T10:19:29.983937Z 2022-01-17T10:19:35.794624Z 2022-01-17T10:19:28.702014Z 2022-01-17T10:19:34.927372Z 2022-01-17T10:19:27.285941Z 2022-01-17T10:19:30.038552Z 2022-01-17T10:19:27.121020Z 2022-01-17T10:19:22.381314Z 2022-01-17T10:19:20.137302Z 2022-01-17T10:19:24.022847Z 2022-01-17T10:24:43.878998Z 2022-01-17T10:24:50.349534Z 2022-01-17T10:24:44.799803Z 2022-01-17T10:24:51.127514Z 2022-01-17T10:24:41.390664Z 2022-01-17T10:24:49.057611Z 2022-01-17T10:24:43.062670Z 2022-01-17T10:24:50.559161Z 2022-01-17T10:24:40.888936Z 2022-01-17T10:24:53.153678Z 2022-01-17T10:24:40.385999Z 2022-01-17T10:24:48.219512Z 2022-01-17T10:24:42.810905Z 2022-01-17T10:24:50.593898Z 2022-01-17T10:24:44.883125Z 2022-01-17T10:24:44.999597Z 2022-01-17T10:24:43.472953Z 2022-01-17T10:24:52.937329Z 2022-01-17T10:24:42.329845Z 2022-01-17T10:24:43.991175Z 2022-01-17T10:24:41.141043Z 2022-01-17T10:24:46.958556Z 2022-01-17T10:24:38.688435Z 2022-01-17T10:24:46.446948Z 2022-01-17T10:24:38.368989Z 2022-01-17T10:24:45.519710Z 2022-01-17T10:24:39.282678Z 2022-01-17T10:24:42.667614Z 2022-01-17T10:24:36.675864Z 2022-01-17T10:24:34.125068Z 2022-01-17T10:24:40.095358Z 2022-01-17T10:24:43.991175Z 2022-01-17T10:24:36.949946Z 2022-01-17T10:24:49.089675Z 2022-01-17T10:24:27.997933Z 2022-01-17T10:24:38.640572Z 2022-01-17T10:24:36.119988Z 2022-01-17T10:24:41.384436Z 2022-01-17T10:24:36.125412Z 2022-01-17T10:24:38.640572Z 2022-01-17T10:24:34.305555Z 2022-01-17T10:24:46.958556Z 2022-01-17T10:24:31.362609Z 2022-01-17T10:24:44.243605Z 2022-01-17T10:24:34.542200Z 2022-01-17T10:24:41.647643Z 2022-01-17T10:24:28.670883Z 2022-01-17T10:24:35.083691Z 2022-01-17T10:24:27.763085Z 2022-01-17T10:24:42.606538Z 2022-01-17T10:24:33.113258Z 2022-01-17T10:24:39.140888Z 2022-01-17T10:24:21.989666Z 2022-01-17T10:24:35.084754Z 2022-01-17T10:24:26.345800Z 2022-01-17T10:24:27.013435Z 2022-01-17T10:24:26.875175Z 2022-01-17T10:24:31.510909Z 2022-01-17T10:24:19.177372Z 2022-01-17T10:24:33.103074Z 2022-01-17T10:24:19.904058Z 2022-01-17T10:24:27.476018Z 2022-01-17T10:24:32.351166Z 2022-01-17T10:24:12.519401Z 2022-01-17T10:24:21.412621Z 2022-01-17T10:24:27.944776Z 2022-01-17T10:24:12.244127Z 2022-01-17T10:24:30.277537Z 2022-01-17T10:24:10.519419Z 2022-01-17T10:24:16.064211Z 2022-01-17T10:24:10.300362Z 2022-01-17T10:24:18.562057Z 2022-01-17T10:23:57.102233Z 2022-01-17T10:24:21.792503Z 2022-01-17T10:23:53.569012Z 2022-01-17T10:24:19.634645Z 2022-01-17T10:24:02.018912Z 2022-01-17T10:24:29.188598Z 2022-01-17T10:23:58.290877Z 2022-01-17T10:24:15.806856Z 2022-01-17T10:23:50.670457Z 2022-01-17T10:24:11.797823Z 2022-01-17T10:23:48.386066Z 2022-01-17T10:23:35.201643Z 2022-01-17T10:23:28.827398Z 2022-01-17T10:23:38.390124Z 2022-01-17T10:23:29.660283Z 2022-01-17T10:23:40.306890Z 2022-01-17T10:23:28.352922Z 2022-01-17T10:23:35.584510Z 2022-01-17T10:23:28.108514Z 2022-01-17T10:23:34.013636Z 2022-01-17T10:23:26.367744Z 2022-01-17T10:23:33.354988Z 2022-01-17T10:23:26.281118Z 2022-01-17T10:23:29.226097Z 2022-01-17T10:23:26.618915Z 2022-01-17T10:23:21.197424Z 2022-01-17T10:23:19.325802Z 2022-01-17T10:23:18.418825Z 2022-01-17T10:29:07.776506Z 2022-01-17T10:29:01.304374Z 2022-01-17T10:29:09.178479Z 2022-01-17T10:28:57.306106Z 2022-01-17T10:29:09.747517Z 2022-01-17T10:28:55.534946Z 2022-01-17T10:29:09.747517Z 2022-01-17T10:28:58.543704Z 2022-01-17T10:29:09.747517Z 2022-01-17T10:28:58.543704Z 2022-01-17T10:29:00.713903Z 2022-01-17T10:29:00.059195Z 2022-01-17T10:29:05.513739Z 2022-01-17T10:28:55.531103Z 2022-01-17T10:29:05.247717Z 2022-01-17T10:29:01.026566Z 2022-01-17T10:29:10.846432Z 2022-01-17T10:28:59.559098Z 2022-01-17T10:29:08.028413Z 2022-01-17T10:28:51.970220Z 2022-01-17T10:29:06.134524Z 2022-01-17T10:29:04.281830Z 2022-01-17T10:28:48.888620Z 2022-01-17T10:28:53.753396Z 2022-01-17T10:29:06.134524Z 2022-01-17T10:28:52.523195Z 2022-01-17T10:29:07.006426Z 2022-01-17T10:28:56.428788Z 2022-01-17T10:29:02.766319Z 2022-01-17T10:29:02.552777Z 2022-01-17T10:28:52.269000Z 2022-01-17T10:28:54.004088Z 2022-01-17T10:28:58.778219Z 2022-01-17T10:28:48.443146Z 2022-01-17T10:28:58.116973Z 2022-01-17T10:28:54.257276Z 2022-01-17T10:28:55.235845Z 2022-01-17T10:29:02.766319Z 2022-01-17T10:28:43.047207Z 2022-01-17T10:28:57.173635Z 2022-01-17T10:28:53.753396Z 2022-01-17T10:28:59.233780Z 2022-01-17T10:28:54.004088Z 2022-01-17T10:28:57.008225Z 2022-01-17T10:28:42.145487Z 2022-01-17T10:28:56.086500Z 2022-01-17T10:28:43.407309Z 2022-01-17T10:28:54.796504Z 2022-01-17T10:28:48.440172Z 2022-01-17T10:28:54.500011Z 2022-01-17T10:28:43.489328Z 2022-01-17T10:28:48.191070Z 2022-01-17T10:28:43.890829Z 2022-01-17T10:28:43.407309Z 2022-01-17T10:28:44.996729Z 2022-01-17T10:28:53.170578Z 2022-01-17T10:28:36.134276Z 2022-01-17T10:28:48.886616Z 2022-01-17T10:28:43.407309Z 2022-01-17T10:28:50.106610Z 2022-01-17T10:28:35.240752Z 2022-01-17T10:28:45.844665Z 2022-01-17T10:28:34.724558Z 2022-01-17T10:28:39.754454Z 2022-01-17T10:28:32.192648Z 2022-01-17T10:28:45.009905Z 2022-01-17T10:28:37.390187Z 2022-01-17T10:28:29.418395Z 2022-01-17T10:28:30.161982Z 2022-01-17T10:28:30.872129Z 2022-01-17T10:28:41.737593Z 2022-01-17T10:28:33.247851Z 2022-01-17T10:28:31.053277Z 2022-01-17T10:28:34.959056Z 2022-01-17T10:28:20.856956Z 2022-01-17T10:28:34.449604Z 2022-01-17T10:28:24.765408Z 2022-01-17T10:28:38.466671Z 2022-01-17T10:28:13.383019Z 2022-01-17T10:28:27.264609Z 2022-01-17T10:28:03.884500Z 2022-01-17T10:28:24.266957Z 2022-01-17T10:27:59.494967Z 2022-01-17T10:28:15.834762Z 2022-01-17T10:28:02.641206Z 2022-01-17T10:28:12.075918Z 2022-01-17T10:27:54.300787Z 2022-01-17T10:27:49.830748Z 2022-01-17T10:27:42.488056Z 2022-01-17T10:27:45.679875Z 2022-01-17T10:27:41.872998Z 2022-01-17T10:27:46.645647Z 2022-01-17T10:27:41.537899Z 2022-01-17T10:27:47.757481Z 2022-01-17T10:27:41.285786Z 2022-01-17T10:27:42.222372Z 2022-01-17T10:27:36.340462Z 2022-01-17T10:27:37.603979Z 2022-01-17T10:27:34.242873Z 2022-01-17T10:27:36.040441Z", + "event_last": "2022-01-17T10:11:16.706328Z 2022-01-17T10:10:40.816921Z 2022-01-17T10:11:28.742337Z 2022-01-17T10:10:38.410972Z 2022-01-17T10:11:21.440285Z 2022-01-17T10:10:35.944736Z 2022-01-17T10:11:27.246402Z 2022-01-17T10:10:37.568842Z 2022-01-17T10:11:21.169401Z 2022-01-17T10:10:37.259481Z 2022-01-17T10:11:11.854353Z 2022-01-17T10:10:45.035866Z 2022-01-17T10:11:05.502336Z 2022-01-17T10:13:05.180416Z 2022-01-17T10:11:19.730411Z 2022-01-17T10:11:19.562861Z 2022-01-17T10:11:13.029389Z 2022-01-17T10:12:55.502639Z 2022-01-17T10:11:06.814891Z 2022-01-17T10:12:00.129227Z 2022-01-17T10:11:04.454645Z 2022-01-17T10:12:18.131024Z 2022-01-17T10:10:42.564836Z 2022-01-17T10:12:27.509360Z 2022-01-17T10:10:47.924426Z 2022-01-17T10:12:09.388156Z 2022-01-17T10:10:44.585492Z 2022-01-17T10:12:38.036118Z 2022-01-17T10:10:40.221313Z 2022-01-17T10:12:46.682106Z 2022-01-17T10:10:38.324459Z 2022-01-17T10:11:51.117985Z 2022-01-17T10:10:32.708413Z 2022-01-17T10:10:36.525206Z 2022-01-17T10:10:25.345675Z 2022-01-17T10:11:57.819782Z 2022-01-17T10:11:46.643944Z 2022-01-17T10:11:54.976173Z 2022-01-17T10:11:48.017804Z 2022-01-17T10:11:57.470391Z 2022-01-17T10:11:45.840843Z 2022-01-17T10:11:58.597060Z 2022-01-17T10:11:47.620867Z 2022-01-17T10:11:59.195807Z 2022-01-17T10:11:46.086362Z 2022-01-17T10:11:57.226093Z 2022-01-17T10:11:48.213884Z 2022-01-17T10:11:49.086753Z 2022-01-17T10:11:45.019155Z 2022-01-17T10:11:58.815977Z 2022-01-17T10:11:48.808897Z 2022-01-17T10:11:44.863466Z 2022-01-17T10:11:56.613339Z 2022-01-17T10:11:43.032773Z 2022-01-17T10:11:56.834760Z 2022-01-17T10:11:42.463182Z 2022-01-17T10:11:53.115568Z 2022-01-17T10:11:40.120790Z 2022-01-17T10:11:57.492558Z 2022-01-17T10:11:41.266780Z 2022-01-17T10:11:55.525933Z 2022-01-17T10:11:39.710270Z 2022-01-17T10:11:56.251017Z 2022-01-17T10:11:39.445783Z 2022-01-17T10:11:51.599884Z 2022-01-17T10:11:38.109760Z 2022-01-17T10:11:53.685410Z 2022-01-17T10:11:38.045241Z 2022-01-17T10:11:56.013416Z 2022-01-17T10:11:35.127791Z 2022-01-17T10:11:54.724287Z 2022-01-17T10:11:40.196461Z 2022-01-17T10:11:48.168315Z 2022-01-17T10:11:34.519936Z 2022-01-17T10:11:51.957356Z 2022-01-17T10:11:28.992736Z 2022-01-17T10:11:47.647345Z 2022-01-17T10:11:26.613757Z 2022-01-17T10:11:46.987108Z 2022-01-17T10:11:26.462628Z 2022-01-17T10:11:43.292760Z 2022-01-17T10:11:27.800872Z 2022-01-17T10:11:46.262105Z 2022-01-17T10:11:30.612686Z 2022-01-17T10:11:44.619308Z 2022-01-17T10:11:27.268350Z 2022-01-17T10:11:48.534579Z 2022-01-17T10:11:26.158648Z 2022-01-17T10:11:34.964199Z 2022-01-17T10:11:18.301977Z 2022-01-17T10:11:43.438386Z 2022-01-17T10:11:08.090779Z 2022-01-17T10:11:39.523595Z 2022-01-17T10:11:05.672833Z 2022-01-17T10:11:45.273820Z 2022-01-17T10:10:57.279819Z 2022-01-17T10:11:36.703348Z 2022-01-17T10:10:52.473946Z 2022-01-17T10:11:22.270383Z 2022-01-17T10:10:38.078298Z 2022-01-17T10:16:38.133757Z 2022-01-17T10:16:42.730892Z 2022-01-17T10:16:41.514501Z 2022-01-17T10:16:42.247369Z 2022-01-17T10:16:38.613856Z 2022-01-17T10:16:43.078010Z 2022-01-17T10:16:39.247792Z 2022-01-17T10:16:45.551545Z 2022-01-17T10:16:39.321478Z 2022-01-17T10:16:43.360052Z 2022-01-17T10:16:38.204820Z 2022-01-17T10:16:41.040874Z 2022-01-17T10:16:39.103747Z 2022-01-17T10:16:42.730892Z 2022-01-17T10:16:38.758954Z 2022-01-17T10:16:42.730892Z 2022-01-17T10:16:37.308690Z 2022-01-17T10:16:40.164969Z 2022-01-17T10:16:36.820996Z 2022-01-17T10:16:40.597247Z 2022-01-17T10:16:37.021237Z 2022-01-17T10:16:40.442240Z 2022-01-17T10:16:38.289827Z 2022-01-17T10:16:34.621742Z 2022-01-17T10:16:26.636202Z 2022-01-17T10:16:33.944208Z 2022-01-17T10:16:31.287753Z 2022-01-17T10:16:37.264046Z 2022-01-17T10:16:30.874054Z 2022-01-17T10:16:36.955477Z 2022-01-17T10:16:27.406805Z 2022-01-17T10:16:33.951973Z 2022-01-17T10:16:30.401734Z 2022-01-17T10:16:37.224640Z 2022-01-17T10:16:28.809773Z 2022-01-17T10:16:31.815839Z 2022-01-17T10:16:30.122835Z 2022-01-17T10:16:29.125417Z 2022-01-17T10:16:25.691941Z 2022-01-17T10:16:33.054041Z 2022-01-17T10:16:20.039731Z 2022-01-17T10:16:32.912049Z 2022-01-17T10:16:31.434192Z 2022-01-17T10:16:26.861689Z 2022-01-17T10:16:25.690633Z 2022-01-17T10:16:30.686598Z 2022-01-17T10:16:19.057763Z 2022-01-17T10:16:19.988856Z 2022-01-17T10:16:20.279786Z 2022-01-17T10:16:29.205564Z 2022-01-17T10:16:24.920744Z 2022-01-17T10:16:33.177381Z 2022-01-17T10:16:12.578565Z 2022-01-17T10:16:31.182826Z 2022-01-17T10:16:16.753848Z 2022-01-17T10:16:25.220795Z 2022-01-17T10:16:22.975741Z 2022-01-17T10:16:20.304099Z 2022-01-17T10:16:17.972440Z 2022-01-17T10:16:26.655343Z 2022-01-17T10:16:09.392825Z 2022-01-17T10:16:24.711583Z 2022-01-17T10:16:18.532772Z 2022-01-17T10:16:29.498527Z 2022-01-17T10:16:03.384786Z 2022-01-17T10:16:17.416956Z 2022-01-17T10:16:07.393786Z 2022-01-17T10:16:17.230600Z 2022-01-17T10:16:09.732090Z 2022-01-17T10:16:13.127378Z 2022-01-17T10:16:09.371808Z 2022-01-17T10:16:20.395193Z 2022-01-17T10:15:59.815158Z 2022-01-17T10:16:12.148723Z 2022-01-17T10:15:49.472778Z 2022-01-17T10:16:03.610034Z 2022-01-17T10:16:03.909854Z 2022-01-17T10:16:05.435388Z 2022-01-17T10:15:57.309252Z 2022-01-17T10:16:00.341396Z 2022-01-17T10:15:41.724741Z 2022-01-17T10:16:06.746419Z 2022-01-17T10:15:40.319719Z 2022-01-17T10:15:50.662374Z 2022-01-17T10:15:25.622160Z 2022-01-17T10:15:25.626043Z 2022-01-17T10:15:22.352722Z 2022-01-17T10:15:25.961215Z 2022-01-17T10:15:21.290317Z 2022-01-17T10:15:21.971628Z 2022-01-17T10:15:20.119685Z 2022-01-17T10:15:22.381155Z 2022-01-17T10:15:18.993790Z 2022-01-17T10:15:19.938120Z 2022-01-17T10:15:15.714854Z 2022-01-17T10:15:16.965016Z 2022-01-17T10:15:12.497385Z 2022-01-17T10:15:12.821727Z 2022-01-17T10:15:11.286331Z 2022-01-17T10:15:10.945373Z 2022-01-17T10:20:52.569214Z 2022-01-17T10:20:53.489411Z 2022-01-17T10:20:47.463592Z 2022-01-17T10:20:52.113198Z 2022-01-17T10:20:50.120426Z 2022-01-17T10:20:53.935171Z 2022-01-17T10:20:50.072840Z 2022-01-17T10:20:48.884020Z 2022-01-17T10:20:49.142917Z 2022-01-17T10:20:53.228870Z 2022-01-17T10:20:48.801256Z 2022-01-17T10:20:49.327627Z 2022-01-17T10:20:50.163866Z 2022-01-17T10:20:53.489411Z 2022-01-17T10:20:44.538779Z 2022-01-17T10:20:47.678080Z 2022-01-17T10:20:48.565626Z 2022-01-17T10:20:50.089263Z 2022-01-17T10:20:48.118142Z 2022-01-17T10:20:46.259846Z 2022-01-17T10:20:50.549346Z 2022-01-17T10:20:43.024938Z 2022-01-17T10:20:51.375097Z 2022-01-17T10:20:39.133440Z 2022-01-17T10:20:49.682876Z 2022-01-17T10:20:46.691422Z 2022-01-17T10:20:49.694610Z 2022-01-17T10:20:46.704002Z 2022-01-17T10:20:46.259846Z 2022-01-17T10:20:51.505451Z 2022-01-17T10:20:47.616761Z 2022-01-17T10:20:48.126255Z 2022-01-17T10:20:41.257723Z 2022-01-17T10:20:43.695959Z 2022-01-17T10:20:42.525590Z 2022-01-17T10:20:48.188333Z 2022-01-17T10:20:40.310939Z 2022-01-17T10:20:44.789032Z 2022-01-17T10:20:44.106381Z 2022-01-17T10:20:46.488772Z 2022-01-17T10:20:39.319744Z 2022-01-17T10:20:46.533428Z 2022-01-17T10:20:45.192126Z 2022-01-17T10:20:40.998552Z 2022-01-17T10:20:43.782785Z 2022-01-17T10:20:44.446363Z 2022-01-17T10:20:36.043376Z 2022-01-17T10:20:39.419530Z 2022-01-17T10:20:43.607942Z 2022-01-17T10:20:44.907185Z 2022-01-17T10:20:37.032791Z 2022-01-17T10:20:46.364477Z 2022-01-17T10:20:30.689839Z 2022-01-17T10:20:40.820806Z 2022-01-17T10:20:31.670584Z 2022-01-17T10:20:35.036977Z 2022-01-17T10:20:35.251181Z 2022-01-17T10:20:35.775593Z 2022-01-17T10:20:32.616691Z 2022-01-17T10:20:36.854397Z 2022-01-17T10:20:22.032827Z 2022-01-17T10:20:32.969432Z 2022-01-17T10:20:18.731753Z 2022-01-17T10:20:39.459479Z 2022-01-17T10:20:22.602378Z 2022-01-17T10:20:28.979330Z 2022-01-17T10:20:25.090752Z 2022-01-17T10:20:34.790004Z 2022-01-17T10:20:21.096788Z 2022-01-17T10:20:26.579697Z 2022-01-17T10:20:09.750774Z 2022-01-17T10:20:26.309343Z 2022-01-17T10:20:14.656821Z 2022-01-17T10:20:18.386391Z 2022-01-17T10:20:09.876867Z 2022-01-17T10:20:16.980345Z 2022-01-17T10:20:13.446730Z 2022-01-17T10:20:23.174812Z 2022-01-17T10:20:05.592801Z 2022-01-17T10:20:21.568875Z 2022-01-17T10:20:05.330757Z 2022-01-17T10:20:16.523914Z 2022-01-17T10:19:46.087709Z 2022-01-17T10:20:11.557326Z 2022-01-17T10:19:29.925963Z 2022-01-17T10:19:46.665638Z 2022-01-17T10:20:06.096370Z 2022-01-17T10:19:35.006554Z 2022-01-17T10:19:30.424031Z 2022-01-17T10:19:35.604514Z 2022-01-17T10:19:30.776848Z 2022-01-17T10:19:37.728204Z 2022-01-17T10:19:29.529006Z 2022-01-17T10:19:36.709243Z 2022-01-17T10:19:28.724068Z 2022-01-17T10:19:31.539322Z 2022-01-17T10:19:27.826863Z 2022-01-17T10:19:23.474124Z 2022-01-17T10:19:21.240433Z 2022-01-17T10:19:25.022019Z 2022-01-17T10:24:46.245772Z 2022-01-17T10:24:52.436109Z 2022-01-17T10:24:46.150678Z 2022-01-17T10:24:53.224463Z 2022-01-17T10:24:43.682185Z 2022-01-17T10:24:51.596580Z 2022-01-17T10:24:45.228945Z 2022-01-17T10:24:52.269009Z 2022-01-17T10:24:43.071455Z 2022-01-17T10:24:53.829394Z 2022-01-17T10:24:43.071455Z 2022-01-17T10:24:50.898709Z 2022-01-17T10:24:44.513744Z 2022-01-17T10:24:52.476371Z 2022-01-17T10:24:46.336648Z 2022-01-17T10:24:48.977828Z 2022-01-17T10:24:45.131811Z 2022-01-17T10:24:53.600717Z 2022-01-17T10:24:44.799803Z 2022-01-17T10:24:46.813464Z 2022-01-17T10:24:43.645779Z 2022-01-17T10:24:49.827319Z 2022-01-17T10:24:41.537851Z 2022-01-17T10:24:49.969990Z 2022-01-17T10:24:41.348017Z 2022-01-17T10:24:48.302384Z 2022-01-17T10:24:41.910463Z 2022-01-17T10:24:45.637569Z 2022-01-17T10:24:40.032143Z 2022-01-17T10:24:36.790240Z 2022-01-17T10:24:43.523454Z 2022-01-17T10:24:47.161448Z 2022-01-17T10:24:40.718819Z 2022-01-17T10:24:51.372091Z 2022-01-17T10:24:32.022777Z 2022-01-17T10:24:42.073356Z 2022-01-17T10:24:39.607935Z 2022-01-17T10:24:44.053348Z 2022-01-17T10:24:39.010589Z 2022-01-17T10:24:41.299535Z 2022-01-17T10:24:37.444733Z 2022-01-17T10:24:49.896356Z 2022-01-17T10:24:34.060282Z 2022-01-17T10:24:46.906660Z 2022-01-17T10:24:37.349982Z 2022-01-17T10:24:45.797963Z 2022-01-17T10:24:31.745804Z 2022-01-17T10:24:39.199370Z 2022-01-17T10:24:30.844020Z 2022-01-17T10:24:46.899472Z 2022-01-17T10:24:35.361843Z 2022-01-17T10:24:42.548331Z 2022-01-17T10:24:26.851943Z 2022-01-17T10:24:38.461663Z 2022-01-17T10:24:29.428776Z 2022-01-17T10:24:31.970341Z 2022-01-17T10:24:29.902775Z 2022-01-17T10:24:35.312346Z 2022-01-17T10:24:21.639413Z 2022-01-17T10:24:36.620119Z 2022-01-17T10:24:23.707740Z 2022-01-17T10:24:31.612513Z 2022-01-17T10:24:36.422798Z 2022-01-17T10:24:16.627866Z 2022-01-17T10:24:24.308779Z 2022-01-17T10:24:32.745587Z 2022-01-17T10:24:16.476720Z 2022-01-17T10:24:33.645046Z 2022-01-17T10:24:14.804767Z 2022-01-17T10:24:20.765198Z 2022-01-17T10:24:14.118733Z 2022-01-17T10:24:22.797361Z 2022-01-17T10:24:02.536941Z 2022-01-17T10:24:25.844873Z 2022-01-17T10:23:59.303304Z 2022-01-17T10:24:24.487112Z 2022-01-17T10:24:05.718315Z 2022-01-17T10:24:34.168373Z 2022-01-17T10:24:03.099804Z 2022-01-17T10:24:20.185352Z 2022-01-17T10:23:55.072604Z 2022-01-17T10:24:15.820097Z 2022-01-17T10:23:52.544863Z 2022-01-17T10:23:37.036094Z 2022-01-17T10:23:30.026741Z 2022-01-17T10:23:41.067015Z 2022-01-17T10:23:30.519181Z 2022-01-17T10:23:43.547245Z 2022-01-17T10:23:29.416860Z 2022-01-17T10:23:37.941098Z 2022-01-17T10:23:29.223059Z 2022-01-17T10:23:36.584231Z 2022-01-17T10:23:27.482760Z 2022-01-17T10:23:35.639503Z 2022-01-17T10:23:27.598751Z 2022-01-17T10:23:30.831187Z 2022-01-17T10:23:27.482760Z 2022-01-17T10:23:22.448521Z 2022-01-17T10:23:20.568452Z 2022-01-17T10:23:19.446867Z 2022-01-17T10:29:09.652400Z 2022-01-17T10:29:02.630985Z 2022-01-17T10:29:10.832206Z 2022-01-17T10:28:59.192712Z 2022-01-17T10:29:10.846432Z 2022-01-17T10:28:58.759987Z 2022-01-17T10:29:11.345386Z 2022-01-17T10:28:59.869723Z 2022-01-17T10:29:11.095663Z 2022-01-17T10:29:00.703450Z 2022-01-17T10:29:03.456795Z 2022-01-17T10:29:01.552426Z 2022-01-17T10:29:07.498049Z 2022-01-17T10:28:58.563843Z 2022-01-17T10:29:07.119957Z 2022-01-17T10:29:02.336260Z 2022-01-17T10:29:13.163399Z 2022-01-17T10:29:01.589188Z 2022-01-17T10:29:10.116304Z 2022-01-17T10:28:55.339886Z 2022-01-17T10:29:08.763594Z 2022-01-17T10:29:06.715794Z 2022-01-17T10:28:51.698522Z 2022-01-17T10:28:56.601984Z 2022-01-17T10:29:08.732078Z 2022-01-17T10:28:55.717953Z 2022-01-17T10:29:09.178479Z 2022-01-17T10:28:58.818850Z 2022-01-17T10:29:05.838654Z 2022-01-17T10:29:05.161394Z 2022-01-17T10:28:54.558747Z 2022-01-17T10:28:56.180766Z 2022-01-17T10:29:02.858497Z 2022-01-17T10:28:50.478782Z 2022-01-17T10:29:02.166656Z 2022-01-17T10:28:57.163306Z 2022-01-17T10:28:58.989453Z 2022-01-17T10:29:06.158373Z 2022-01-17T10:28:46.611495Z 2022-01-17T10:29:01.094664Z 2022-01-17T10:28:56.241798Z 2022-01-17T10:29:03.539479Z 2022-01-17T10:28:56.395406Z 2022-01-17T10:28:59.407339Z 2022-01-17T10:28:45.323331Z 2022-01-17T10:29:00.003361Z 2022-01-17T10:28:47.860753Z 2022-01-17T10:28:57.921963Z 2022-01-17T10:28:50.743746Z 2022-01-17T10:28:59.550435Z 2022-01-17T10:28:47.097917Z 2022-01-17T10:28:51.879412Z 2022-01-17T10:28:48.019991Z 2022-01-17T10:28:46.999633Z 2022-01-17T10:28:48.880757Z 2022-01-17T10:28:55.987938Z 2022-01-17T10:28:39.748323Z 2022-01-17T10:28:52.702350Z 2022-01-17T10:28:46.751971Z 2022-01-17T10:28:54.613365Z 2022-01-17T10:28:39.558771Z 2022-01-17T10:28:49.208427Z 2022-01-17T10:28:39.359809Z 2022-01-17T10:28:42.980352Z 2022-01-17T10:28:35.212749Z 2022-01-17T10:28:49.686729Z 2022-01-17T10:28:40.930327Z 2022-01-17T10:28:33.779354Z 2022-01-17T10:28:34.113761Z 2022-01-17T10:28:35.013361Z 2022-01-17T10:28:45.112852Z 2022-01-17T10:28:38.114338Z 2022-01-17T10:28:35.368741Z 2022-01-17T10:28:39.802349Z 2022-01-17T10:28:25.137656Z 2022-01-17T10:28:39.562537Z 2022-01-17T10:28:28.352803Z 2022-01-17T10:28:42.243700Z 2022-01-17T10:28:19.031923Z 2022-01-17T10:28:31.795335Z 2022-01-17T10:28:08.034738Z 2022-01-17T10:28:27.856399Z 2022-01-17T10:28:03.268720Z 2022-01-17T10:28:20.831813Z 2022-01-17T10:28:07.301833Z 2022-01-17T10:28:18.392582Z 2022-01-17T10:27:58.042595Z 2022-01-17T10:27:52.009418Z 2022-01-17T10:27:44.900041Z 2022-01-17T10:27:47.512201Z 2022-01-17T10:27:42.788460Z 2022-01-17T10:27:48.174421Z 2022-01-17T10:27:41.881149Z 2022-01-17T10:27:49.867922Z 2022-01-17T10:27:41.898892Z 2022-01-17T10:27:43.619407Z 2022-01-17T10:27:39.231727Z 2022-01-17T10:27:38.812347Z 2022-01-17T10:27:36.340639Z 2022-01-17T10:27:37.140543Z", + "failed_job_ids": " []", + "finished": "2022-01-17T10:11:23.404604Z 2022-01-17T10:10:42.167449Z 2022-01-17T10:11:34.310506Z 2022-01-17T10:10:42.613654Z 2022-01-17T10:11:27.690450Z 2022-01-17T10:10:40.290977Z 2022-01-17T10:11:32.834145Z 2022-01-17T10:10:41.102309Z 2022-01-17T10:11:27.621282Z 2022-01-17T10:10:41.706907Z 2022-01-17T10:11:17.472178Z 2022-01-17T10:10:47.731713Z 2022-01-17T10:11:10.662291Z 2022-01-17T10:13:07.792332Z 2022-01-17T10:11:26.710003Z 2022-01-17T10:11:24.634568Z 2022-01-17T10:11:20.768216Z 2022-01-17T10:12:59.430719Z 2022-01-17T10:11:12.819979Z 2022-01-17T10:12:03.380599Z 2022-01-17T10:11:08.825413Z 2022-01-17T10:12:22.046999Z 2022-01-17T10:10:47.751836Z 2022-01-17T10:12:31.434119Z 2022-01-17T10:10:52.729658Z 2022-01-17T10:12:12.555772Z 2022-01-17T10:10:50.049402Z 2022-01-17T10:12:41.282415Z 2022-01-17T10:10:44.896218Z 2022-01-17T10:12:50.659316Z 2022-01-17T10:10:43.617084Z 2022-01-17T10:11:54.970314Z 2022-01-17T10:10:36.485195Z 2022-01-17T10:10:40.399209Z 2022-01-17T10:10:29.120247Z 2022-01-17T10:12:01.747906Z 2022-01-17T10:11:50.914018Z 2022-01-17T10:12:00.628001Z 2022-01-17T10:11:52.136780Z 2022-01-17T10:12:02.203410Z 2022-01-17T10:11:50.823097Z 2022-01-17T10:12:02.411821Z 2022-01-17T10:11:51.291674Z 2022-01-17T10:12:03.013792Z 2022-01-17T10:11:50.616201Z 2022-01-17T10:12:02.017053Z 2022-01-17T10:11:52.652929Z 2022-01-17T10:11:53.240292Z 2022-01-17T10:11:48.639711Z 2022-01-17T10:12:02.881581Z 2022-01-17T10:11:53.142271Z 2022-01-17T10:11:48.231321Z 2022-01-17T10:12:01.467321Z 2022-01-17T10:11:47.427636Z 2022-01-17T10:12:00.881040Z 2022-01-17T10:11:46.784390Z 2022-01-17T10:11:57.583581Z 2022-01-17T10:11:44.895883Z 2022-01-17T10:12:02.266517Z 2022-01-17T10:11:45.243859Z 2022-01-17T10:12:01.097588Z 2022-01-17T10:11:45.499663Z 2022-01-17T10:12:01.512648Z 2022-01-17T10:11:44.338730Z 2022-01-17T10:11:57.061501Z 2022-01-17T10:11:43.548343Z 2022-01-17T10:11:59.125303Z 2022-01-17T10:11:42.538655Z 2022-01-17T10:12:01.005118Z 2022-01-17T10:11:39.768878Z 2022-01-17T10:11:59.369298Z 2022-01-17T10:11:45.530932Z 2022-01-17T10:11:53.227687Z 2022-01-17T10:11:39.571884Z 2022-01-17T10:11:56.568749Z 2022-01-17T10:11:33.793234Z 2022-01-17T10:11:52.551551Z 2022-01-17T10:11:32.947174Z 2022-01-17T10:11:53.198049Z 2022-01-17T10:11:30.798757Z 2022-01-17T10:11:47.965377Z 2022-01-17T10:11:32.509299Z 2022-01-17T10:11:52.646218Z 2022-01-17T10:11:34.710674Z 2022-01-17T10:11:51.160744Z 2022-01-17T10:11:31.496764Z 2022-01-17T10:11:54.032785Z 2022-01-17T10:11:31.249913Z 2022-01-17T10:11:40.645294Z 2022-01-17T10:11:23.597417Z 2022-01-17T10:11:49.368317Z 2022-01-17T10:11:14.274046Z 2022-01-17T10:11:44.259768Z 2022-01-17T10:11:10.539078Z 2022-01-17T10:11:50.951949Z 2022-01-17T10:11:01.724495Z 2022-01-17T10:11:41.706917Z 2022-01-17T10:10:57.140430Z 2022-01-17T10:11:27.514200Z 2022-01-17T10:10:42.255977Z 2022-01-17T10:16:42.644288Z 2022-01-17T10:16:46.899027Z 2022-01-17T10:16:43.702719Z 2022-01-17T10:16:46.938203Z 2022-01-17T10:16:43.227057Z 2022-01-17T10:16:47.303319Z 2022-01-17T10:16:43.518041Z 2022-01-17T10:16:47.236039Z 2022-01-17T10:16:43.847866Z 2022-01-17T10:16:47.626008Z 2022-01-17T10:16:42.661111Z 2022-01-17T10:16:45.024162Z 2022-01-17T10:16:43.018746Z 2022-01-17T10:16:46.567524Z 2022-01-17T10:16:43.339903Z 2022-01-17T10:16:46.206908Z 2022-01-17T10:16:41.182236Z 2022-01-17T10:16:44.804060Z 2022-01-17T10:16:41.078359Z 2022-01-17T10:16:45.107701Z 2022-01-17T10:16:41.444435Z 2022-01-17T10:16:44.667080Z 2022-01-17T10:16:42.678408Z 2022-01-17T10:16:38.958944Z 2022-01-17T10:16:31.875583Z 2022-01-17T10:16:38.510517Z 2022-01-17T10:16:36.710935Z 2022-01-17T10:16:41.996243Z 2022-01-17T10:16:36.437902Z 2022-01-17T10:16:42.429063Z 2022-01-17T10:16:32.426923Z 2022-01-17T10:16:38.773994Z 2022-01-17T10:16:34.646643Z 2022-01-17T10:16:41.988118Z 2022-01-17T10:16:34.018720Z 2022-01-17T10:16:37.368627Z 2022-01-17T10:16:36.027726Z 2022-01-17T10:16:35.379463Z 2022-01-17T10:16:31.117880Z 2022-01-17T10:16:39.075326Z 2022-01-17T10:16:25.728096Z 2022-01-17T10:16:38.684196Z 2022-01-17T10:16:36.334693Z 2022-01-17T10:16:32.295499Z 2022-01-17T10:16:31.829149Z 2022-01-17T10:16:36.866271Z 2022-01-17T10:16:23.551453Z 2022-01-17T10:16:25.052274Z 2022-01-17T10:16:25.890209Z 2022-01-17T10:16:35.460220Z 2022-01-17T10:16:29.563148Z 2022-01-17T10:16:37.501374Z 2022-01-17T10:16:17.619632Z 2022-01-17T10:16:36.226343Z 2022-01-17T10:16:23.076645Z 2022-01-17T10:16:30.741048Z 2022-01-17T10:16:28.716871Z 2022-01-17T10:16:26.801517Z 2022-01-17T10:16:23.590739Z 2022-01-17T10:16:32.720355Z 2022-01-17T10:16:15.799697Z 2022-01-17T10:16:29.737552Z 2022-01-17T10:16:24.738958Z 2022-01-17T10:16:35.611718Z 2022-01-17T10:16:07.649804Z 2022-01-17T10:16:24.240183Z 2022-01-17T10:16:12.712432Z 2022-01-17T10:16:23.065891Z 2022-01-17T10:16:15.687873Z 2022-01-17T10:16:18.860527Z 2022-01-17T10:16:15.540624Z 2022-01-17T10:16:26.558352Z 2022-01-17T10:16:04.891539Z 2022-01-17T10:16:19.292173Z 2022-01-17T10:15:55.018792Z 2022-01-17T10:16:09.911323Z 2022-01-17T10:16:09.922617Z 2022-01-17T10:16:11.203443Z 2022-01-17T10:16:03.116787Z 2022-01-17T10:16:05.740209Z 2022-01-17T10:15:46.064612Z 2022-01-17T10:16:14.165700Z 2022-01-17T10:15:45.670804Z 2022-01-17T10:15:58.150717Z 2022-01-17T10:15:27.412801Z 2022-01-17T10:15:30.533298Z 2022-01-17T10:15:26.345663Z 2022-01-17T10:15:30.609071Z 2022-01-17T10:15:24.709373Z 2022-01-17T10:15:26.195002Z 2022-01-17T10:15:24.158406Z 2022-01-17T10:15:26.418504Z 2022-01-17T10:15:23.043620Z 2022-01-17T10:15:24.533079Z 2022-01-17T10:15:17.143681Z 2022-01-17T10:15:18.283517Z 2022-01-17T10:15:16.782661Z 2022-01-17T10:15:16.586839Z 2022-01-17T10:15:14.811459Z 2022-01-17T10:15:12.971610Z 2022-01-17T10:20:54.392482Z 2022-01-17T10:20:57.373399Z 2022-01-17T10:20:52.551462Z 2022-01-17T10:20:57.057160Z 2022-01-17T10:20:54.145057Z 2022-01-17T10:20:58.079418Z 2022-01-17T10:20:53.780249Z 2022-01-17T10:20:54.029110Z 2022-01-17T10:20:53.507851Z 2022-01-17T10:20:56.818543Z 2022-01-17T10:20:52.745762Z 2022-01-17T10:20:53.876789Z 2022-01-17T10:20:54.223991Z 2022-01-17T10:20:57.463824Z 2022-01-17T10:20:50.530080Z 2022-01-17T10:20:53.019017Z 2022-01-17T10:20:53.311074Z 2022-01-17T10:20:54.453639Z 2022-01-17T10:20:52.513058Z 2022-01-17T10:20:50.955338Z 2022-01-17T10:20:56.251733Z 2022-01-17T10:20:50.083951Z 2022-01-17T10:20:55.916825Z 2022-01-17T10:20:44.201941Z 2022-01-17T10:20:54.031399Z 2022-01-17T10:20:51.498005Z 2022-01-17T10:20:54.377355Z 2022-01-17T10:20:51.268899Z 2022-01-17T10:20:51.701131Z 2022-01-17T10:20:55.878319Z 2022-01-17T10:20:52.442204Z 2022-01-17T10:20:53.313539Z 2022-01-17T10:20:47.228577Z 2022-01-17T10:20:48.844331Z 2022-01-17T10:20:47.239541Z 2022-01-17T10:20:53.884725Z 2022-01-17T10:20:46.294619Z 2022-01-17T10:20:49.906785Z 2022-01-17T10:20:49.477181Z 2022-01-17T10:20:51.632267Z 2022-01-17T10:20:44.395294Z 2022-01-17T10:20:53.018725Z 2022-01-17T10:20:49.857322Z 2022-01-17T10:20:46.943909Z 2022-01-17T10:20:49.859301Z 2022-01-17T10:20:50.327494Z 2022-01-17T10:20:42.169952Z 2022-01-17T10:20:44.177771Z 2022-01-17T10:20:47.881743Z 2022-01-17T10:20:51.497101Z 2022-01-17T10:20:42.660962Z 2022-01-17T10:20:51.054803Z 2022-01-17T10:20:36.367592Z 2022-01-17T10:20:47.110938Z 2022-01-17T10:20:37.199739Z 2022-01-17T10:20:39.860240Z 2022-01-17T10:20:41.134443Z 2022-01-17T10:20:42.587208Z 2022-01-17T10:20:37.689720Z 2022-01-17T10:20:41.971084Z 2022-01-17T10:20:28.176242Z 2022-01-17T10:20:39.732064Z 2022-01-17T10:20:24.956804Z 2022-01-17T10:20:45.316959Z 2022-01-17T10:20:27.460753Z 2022-01-17T10:20:36.108124Z 2022-01-17T10:20:29.969014Z 2022-01-17T10:20:39.506497Z 2022-01-17T10:20:25.874442Z 2022-01-17T10:20:32.127188Z 2022-01-17T10:20:16.420463Z 2022-01-17T10:20:32.007333Z 2022-01-17T10:20:21.786706Z 2022-01-17T10:20:23.873408Z 2022-01-17T10:20:15.708603Z 2022-01-17T10:20:22.830019Z 2022-01-17T10:20:19.038503Z 2022-01-17T10:20:29.090186Z 2022-01-17T10:20:09.648650Z 2022-01-17T10:20:27.466889Z 2022-01-17T10:20:11.168411Z 2022-01-17T10:20:22.832178Z 2022-01-17T10:19:51.164975Z 2022-01-17T10:20:17.197045Z 2022-01-17T10:19:33.996562Z 2022-01-17T10:19:51.997928Z 2022-01-17T10:20:10.937278Z 2022-01-17T10:19:40.597924Z 2022-01-17T10:19:34.776008Z 2022-01-17T10:19:40.760834Z 2022-01-17T10:19:35.113462Z 2022-01-17T10:19:41.850380Z 2022-01-17T10:19:33.655043Z 2022-01-17T10:19:41.636221Z 2022-01-17T10:19:32.095021Z 2022-01-17T10:19:36.336160Z 2022-01-17T10:19:32.416684Z 2022-01-17T10:19:27.603367Z 2022-01-17T10:19:25.235583Z 2022-01-17T10:19:28.965308Z 2022-01-17T10:24:49.809979Z 2022-01-17T10:24:56.233545Z 2022-01-17T10:24:50.042830Z 2022-01-17T10:24:57.404788Z 2022-01-17T10:24:49.703058Z 2022-01-17T10:24:56.477677Z 2022-01-17T10:24:49.845148Z 2022-01-17T10:24:56.846048Z 2022-01-17T10:24:46.712906Z 2022-01-17T10:24:57.549908Z 2022-01-17T10:24:48.452334Z 2022-01-17T10:24:55.112993Z 2022-01-17T10:24:48.242288Z 2022-01-17T10:24:56.375822Z 2022-01-17T10:24:50.377711Z 2022-01-17T10:24:53.876415Z 2022-01-17T10:24:49.426938Z 2022-01-17T10:24:57.867921Z 2022-01-17T10:24:49.412735Z 2022-01-17T10:24:51.495100Z 2022-01-17T10:24:48.374346Z 2022-01-17T10:24:54.596167Z 2022-01-17T10:24:46.324869Z 2022-01-17T10:24:54.415549Z 2022-01-17T10:24:45.247341Z 2022-01-17T10:24:53.387419Z 2022-01-17T10:24:46.052663Z 2022-01-17T10:24:51.841274Z 2022-01-17T10:24:45.614058Z 2022-01-17T10:24:42.809469Z 2022-01-17T10:24:48.642487Z 2022-01-17T10:24:52.499585Z 2022-01-17T10:24:45.056006Z 2022-01-17T10:24:55.972539Z 2022-01-17T10:24:37.103491Z 2022-01-17T10:24:47.507238Z 2022-01-17T10:24:44.556319Z 2022-01-17T10:24:50.801267Z 2022-01-17T10:24:43.428734Z 2022-01-17T10:24:46.246602Z 2022-01-17T10:24:42.714922Z 2022-01-17T10:24:54.181712Z 2022-01-17T10:24:38.482986Z 2022-01-17T10:24:51.957309Z 2022-01-17T10:24:43.262696Z 2022-01-17T10:24:51.052268Z 2022-01-17T10:24:38.213724Z 2022-01-17T10:24:44.580592Z 2022-01-17T10:24:35.888959Z 2022-01-17T10:24:51.766772Z 2022-01-17T10:24:39.723673Z 2022-01-17T10:24:48.799298Z 2022-01-17T10:24:31.100655Z 2022-01-17T10:24:42.810342Z 2022-01-17T10:24:35.970604Z 2022-01-17T10:24:37.734924Z 2022-01-17T10:24:35.843513Z 2022-01-17T10:24:40.783825Z 2022-01-17T10:24:27.824693Z 2022-01-17T10:24:42.972415Z 2022-01-17T10:24:28.485963Z 2022-01-17T10:24:37.297241Z 2022-01-17T10:24:42.106296Z 2022-01-17T10:24:22.035162Z 2022-01-17T10:24:29.211154Z 2022-01-17T10:24:37.118161Z 2022-01-17T10:24:21.893116Z 2022-01-17T10:24:39.122413Z 2022-01-17T10:24:20.849702Z 2022-01-17T10:24:26.023273Z 2022-01-17T10:24:19.696586Z 2022-01-17T10:24:28.710288Z 2022-01-17T10:24:07.919808Z 2022-01-17T10:24:32.086097Z 2022-01-17T10:24:04.488576Z 2022-01-17T10:24:29.949231Z 2022-01-17T10:24:10.978628Z 2022-01-17T10:24:40.235205Z 2022-01-17T10:24:08.495027Z 2022-01-17T10:24:27.817302Z 2022-01-17T10:24:00.854595Z 2022-01-17T10:24:21.767893Z 2022-01-17T10:23:58.383833Z 2022-01-17T10:23:43.355787Z 2022-01-17T10:23:34.122677Z 2022-01-17T10:23:46.199242Z 2022-01-17T10:23:34.891039Z 2022-01-17T10:23:48.530239Z 2022-01-17T10:23:33.587363Z 2022-01-17T10:23:42.651631Z 2022-01-17T10:23:33.317440Z 2022-01-17T10:23:40.810448Z 2022-01-17T10:23:31.567582Z 2022-01-17T10:23:41.536412Z 2022-01-17T10:23:32.343190Z 2022-01-17T10:23:35.049607Z 2022-01-17T10:23:31.680130Z 2022-01-17T10:23:26.117545Z 2022-01-17T10:23:24.854723Z 2022-01-17T10:23:23.806399Z 2022-01-17T10:29:13.898797Z 2022-01-17T10:29:06.595562Z 2022-01-17T10:29:15.201233Z 2022-01-17T10:29:04.261987Z 2022-01-17T10:29:14.927939Z 2022-01-17T10:29:02.285391Z 2022-01-17T10:29:15.581837Z 2022-01-17T10:29:04.417422Z 2022-01-17T10:29:15.203399Z 2022-01-17T10:29:04.946480Z 2022-01-17T10:29:07.997880Z 2022-01-17T10:29:05.422094Z 2022-01-17T10:29:11.649444Z 2022-01-17T10:29:02.713144Z 2022-01-17T10:29:12.104636Z 2022-01-17T10:29:06.700312Z 2022-01-17T10:29:15.664274Z 2022-01-17T10:29:05.809265Z 2022-01-17T10:29:13.936434Z 2022-01-17T10:29:01.050731Z 2022-01-17T10:29:13.559034Z 2022-01-17T10:29:11.791920Z 2022-01-17T10:28:56.832688Z 2022-01-17T10:29:01.199172Z 2022-01-17T10:29:13.552907Z 2022-01-17T10:29:00.673757Z 2022-01-17T10:29:13.559000Z 2022-01-17T10:29:03.563108Z 2022-01-17T10:29:11.103977Z 2022-01-17T10:29:08.924283Z 2022-01-17T10:28:59.751459Z 2022-01-17T10:29:01.415623Z 2022-01-17T10:29:07.421981Z 2022-01-17T10:28:55.648128Z 2022-01-17T10:29:06.412815Z 2022-01-17T10:29:01.901911Z 2022-01-17T10:29:04.495913Z 2022-01-17T10:29:10.282964Z 2022-01-17T10:28:50.971630Z 2022-01-17T10:29:07.903300Z 2022-01-17T10:29:00.558333Z 2022-01-17T10:29:08.490404Z 2022-01-17T10:29:01.710223Z 2022-01-17T10:29:04.990225Z 2022-01-17T10:28:51.271757Z 2022-01-17T10:29:07.121873Z 2022-01-17T10:28:53.330649Z 2022-01-17T10:29:04.917818Z 2022-01-17T10:28:56.006704Z 2022-01-17T10:29:06.114777Z 2022-01-17T10:28:53.268743Z 2022-01-17T10:28:57.251395Z 2022-01-17T10:28:53.782310Z 2022-01-17T10:28:52.666614Z 2022-01-17T10:28:55.012730Z 2022-01-17T10:29:01.081873Z 2022-01-17T10:28:45.992638Z 2022-01-17T10:28:59.403282Z 2022-01-17T10:28:52.189756Z 2022-01-17T10:28:58.512530Z 2022-01-17T10:28:44.933666Z 2022-01-17T10:28:54.902572Z 2022-01-17T10:28:44.229636Z 2022-01-17T10:28:47.559751Z 2022-01-17T10:28:41.484021Z 2022-01-17T10:28:54.861180Z 2022-01-17T10:28:45.994762Z 2022-01-17T10:28:40.078319Z 2022-01-17T10:28:39.893618Z 2022-01-17T10:28:40.348650Z 2022-01-17T10:28:50.308247Z 2022-01-17T10:28:42.823294Z 2022-01-17T10:28:41.409140Z 2022-01-17T10:28:45.716162Z 2022-01-17T10:28:31.356598Z 2022-01-17T10:28:46.962437Z 2022-01-17T10:28:33.521370Z 2022-01-17T10:28:46.701167Z 2022-01-17T10:28:24.343424Z 2022-01-17T10:28:37.054269Z 2022-01-17T10:28:13.940823Z 2022-01-17T10:28:34.416018Z 2022-01-17T10:28:07.478666Z 2022-01-17T10:28:26.629251Z 2022-01-17T10:28:13.378363Z 2022-01-17T10:28:22.496419Z 2022-01-17T10:28:02.408902Z 2022-01-17T10:27:56.776957Z 2022-01-17T10:27:46.579339Z 2022-01-17T10:27:51.441750Z 2022-01-17T10:27:46.870967Z 2022-01-17T10:27:52.769488Z 2022-01-17T10:27:45.401640Z 2022-01-17T10:27:54.198298Z 2022-01-17T10:27:46.306690Z 2022-01-17T10:27:47.961358Z 2022-01-17T10:27:41.853423Z 2022-01-17T10:27:43.435235Z 2022-01-17T10:27:38.812587Z 2022-01-17T10:27:41.209196Z", + "finished_diff": 107.4167054, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 162.039578, + "mean": 52.00625956199999, + "min": 12.905394 + }, + "jobs_events_duration": { + "max": 6.472142, + "mean": 2.9386966219999993, + "min": 0.238696 + }, + "jobs_events_lag": { + "max": -1.318501, + "mean": -4.94184305, + "min": -7.738827 + }, + "jobs_waiting": { + "max": 41.69934, + "mean": 23.69928255600003, + "min": 0.598431 + }, + "modified": "2022-01-17T10:10:23.646552Z 2022-01-17T10:10:23.599678Z 2022-01-17T10:10:23.404603Z 2022-01-17T10:10:23.280891Z 2022-01-17T10:10:23.164802Z 2022-01-17T10:10:23.034441Z 2022-01-17T10:10:22.963603Z 2022-01-17T10:10:22.860442Z 2022-01-17T10:10:22.592279Z 2022-01-17T10:10:22.352079Z 2022-01-17T10:10:22.137057Z 2022-01-17T10:10:22.005408Z 2022-01-17T10:10:21.886892Z 2022-01-17T10:10:21.781751Z 2022-01-17T10:10:21.630627Z 2022-01-17T10:10:21.505678Z 2022-01-17T10:10:21.306164Z 2022-01-17T10:10:21.189644Z 2022-01-17T10:10:21.105833Z 2022-01-17T10:10:21.007153Z 2022-01-17T10:10:20.871974Z 2022-01-17T10:10:17.433100Z 2022-01-17T10:10:17.293323Z 2022-01-17T10:10:17.194083Z 2022-01-17T10:10:17.016573Z 2022-01-17T10:10:16.914822Z 2022-01-17T10:10:16.809218Z 2022-01-17T10:10:16.687904Z 2022-01-17T10:10:15.247339Z 2022-01-17T10:10:15.207570Z 2022-01-17T10:10:15.173479Z 2022-01-17T10:10:14.758305Z 2022-01-17T10:10:14.723297Z 2022-01-17T10:10:14.207977Z 2022-01-17T10:10:13.975386Z 2022-01-17T10:11:05.482776Z 2022-01-17T10:11:05.377368Z 2022-01-17T10:11:05.260932Z 2022-01-17T10:11:05.173799Z 2022-01-17T10:11:05.065869Z 2022-01-17T10:11:04.964332Z 2022-01-17T10:11:04.861778Z 2022-01-17T10:11:04.745777Z 2022-01-17T10:11:04.645777Z 2022-01-17T10:11:04.533792Z 2022-01-17T10:11:04.386766Z 2022-01-17T10:11:04.264794Z 2022-01-17T10:11:04.123065Z 2022-01-17T10:11:03.980101Z 2022-01-17T10:10:43.527328Z 2022-01-17T10:11:03.853002Z 2022-01-17T10:10:43.475375Z 2022-01-17T10:10:43.416365Z 2022-01-17T10:10:43.311592Z 2022-01-17T10:10:43.191607Z 2022-01-17T10:10:43.038514Z 2022-01-17T10:10:42.925256Z 2022-01-17T10:10:42.814747Z 2022-01-17T10:10:42.703393Z 2022-01-17T10:10:42.601427Z 2022-01-17T10:10:42.487315Z 2022-01-17T10:10:42.392942Z 2022-01-17T10:10:42.314360Z 2022-01-17T10:10:42.235464Z 2022-01-17T10:10:42.143320Z 2022-01-17T10:10:42.068394Z 2022-01-17T10:10:41.937412Z 2022-01-17T10:10:41.810780Z 2022-01-17T10:10:41.617447Z 2022-01-17T10:10:41.538705Z 2022-01-17T10:10:41.446445Z 2022-01-17T10:10:41.342743Z 2022-01-17T10:10:41.130835Z 2022-01-17T10:10:40.937658Z 2022-01-17T10:10:40.659221Z 2022-01-17T10:10:40.465648Z 2022-01-17T10:10:40.223422Z 2022-01-17T10:10:39.923224Z 2022-01-17T10:10:39.559291Z 2022-01-17T10:10:39.256608Z 2022-01-17T10:10:38.930023Z 2022-01-17T10:10:38.546663Z 2022-01-17T10:10:38.119940Z 2022-01-17T10:10:37.759797Z 2022-01-17T10:10:37.626405Z 2022-01-17T10:10:37.387737Z 2022-01-17T10:10:37.202696Z 2022-01-17T10:10:37.112257Z 2022-01-17T10:10:37.048215Z 2022-01-17T10:10:36.970887Z 2022-01-17T10:10:36.899992Z 2022-01-17T10:10:36.802828Z 2022-01-17T10:10:36.577751Z 2022-01-17T10:10:36.412772Z 2022-01-17T10:10:36.353882Z 2022-01-17T10:10:36.294579Z 2022-01-17T10:10:36.126085Z 2022-01-17T10:10:35.781597Z 2022-01-17T10:10:23.765662Z 2022-01-17T10:10:23.704762Z 2022-01-17T10:15:49.058790Z 2022-01-17T10:15:48.997094Z 2022-01-17T10:15:48.914684Z 2022-01-17T10:15:48.852103Z 2022-01-17T10:15:48.798634Z 2022-01-17T10:15:48.734729Z 2022-01-17T10:15:48.687614Z 2022-01-17T10:15:48.634781Z 2022-01-17T10:15:48.576879Z 2022-01-17T10:15:48.508724Z 2022-01-17T10:15:48.415627Z 2022-01-17T10:15:48.358829Z 2022-01-17T10:15:48.300595Z 2022-01-17T10:15:48.225689Z 2022-01-17T10:15:48.154448Z 2022-01-17T10:15:48.097734Z 2022-01-17T10:15:48.027729Z 2022-01-17T10:15:47.945733Z 2022-01-17T10:15:47.880614Z 2022-01-17T10:15:47.813481Z 2022-01-17T10:15:47.749804Z 2022-01-17T10:15:47.637642Z 2022-01-17T10:15:47.528659Z 2022-01-17T10:15:47.380056Z 2022-01-17T10:15:20.243268Z 2022-01-17T10:15:20.181301Z 2022-01-17T10:15:20.103559Z 2022-01-17T10:15:20.005425Z 2022-01-17T10:15:19.935632Z 2022-01-17T10:15:19.884026Z 2022-01-17T10:15:19.846228Z 2022-01-17T10:15:19.806161Z 2022-01-17T10:15:19.752736Z 2022-01-17T10:15:19.698665Z 2022-01-17T10:15:19.663182Z 2022-01-17T10:15:19.624746Z 2022-01-17T10:15:19.567611Z 2022-01-17T10:15:19.529645Z 2022-01-17T10:15:19.461528Z 2022-01-17T10:15:19.428775Z 2022-01-17T10:15:19.392608Z 2022-01-17T10:15:19.351962Z 2022-01-17T10:15:19.318493Z 2022-01-17T10:15:19.283163Z 2022-01-17T10:15:19.218977Z 2022-01-17T10:15:19.142295Z 2022-01-17T10:15:18.994550Z 2022-01-17T10:15:18.933446Z 2022-01-17T10:15:18.888944Z 2022-01-17T10:15:18.818282Z 2022-01-17T10:15:18.747174Z 2022-01-17T10:15:18.651348Z 2022-01-17T10:15:18.558473Z 2022-01-17T10:15:18.466436Z 2022-01-17T10:15:18.338502Z 2022-01-17T10:15:18.205280Z 2022-01-17T10:15:18.093342Z 2022-01-17T10:15:17.980219Z 2022-01-17T10:15:17.881220Z 2022-01-17T10:15:17.778260Z 2022-01-17T10:15:17.680237Z 2022-01-17T10:15:17.622091Z 2022-01-17T10:15:17.572337Z 2022-01-17T10:15:17.488862Z 2022-01-17T10:15:17.445394Z 2022-01-17T10:15:17.392965Z 2022-01-17T10:15:17.349264Z 2022-01-17T10:15:17.304579Z 2022-01-17T10:15:17.259004Z 2022-01-17T10:15:17.219574Z 2022-01-17T10:15:17.170547Z 2022-01-17T10:15:17.127584Z 2022-01-17T10:15:17.077414Z 2022-01-17T10:15:16.974658Z 2022-01-17T10:15:16.878821Z 2022-01-17T10:15:16.812368Z 2022-01-17T10:15:16.747368Z 2022-01-17T10:15:16.692665Z 2022-01-17T10:15:16.655714Z 2022-01-17T10:15:16.602928Z 2022-01-17T10:15:16.542562Z 2022-01-17T10:15:16.500029Z 2022-01-17T10:15:16.405766Z 2022-01-17T10:15:16.353192Z 2022-01-17T10:15:04.666116Z 2022-01-17T10:15:04.438789Z 2022-01-17T10:15:04.245848Z 2022-01-17T10:15:03.925602Z 2022-01-17T10:15:03.567792Z 2022-01-17T10:15:03.376531Z 2022-01-17T10:15:03.152824Z 2022-01-17T10:15:02.955767Z 2022-01-17T10:15:02.813956Z 2022-01-17T10:15:02.684158Z 2022-01-17T10:15:00.679426Z 2022-01-17T10:15:00.635066Z 2022-01-17T10:15:00.590147Z 2022-01-17T10:15:00.545437Z 2022-01-17T10:15:00.071362Z 2022-01-17T10:14:59.533772Z 2022-01-17T10:19:54.508649Z 2022-01-17T10:19:54.449704Z 2022-01-17T10:19:54.385714Z 2022-01-17T10:19:54.305868Z 2022-01-17T10:19:54.230030Z 2022-01-17T10:19:54.143794Z 2022-01-17T10:19:54.049818Z 2022-01-17T10:19:53.990280Z 2022-01-17T10:19:53.944814Z 2022-01-17T10:19:53.902617Z 2022-01-17T10:19:53.855706Z 2022-01-17T10:19:53.810650Z 2022-01-17T10:19:53.765658Z 2022-01-17T10:19:53.724721Z 2022-01-17T10:19:28.147955Z 2022-01-17T10:19:53.674850Z 2022-01-17T10:19:53.624705Z 2022-01-17T10:19:28.105248Z 2022-01-17T10:19:53.532843Z 2022-01-17T10:19:28.066776Z 2022-01-17T10:19:27.968968Z 2022-01-17T10:19:27.931417Z 2022-01-17T10:19:27.888276Z 2022-01-17T10:19:27.825229Z 2022-01-17T10:19:27.776356Z 2022-01-17T10:19:27.716264Z 2022-01-17T10:19:27.634009Z 2022-01-17T10:19:27.554268Z 2022-01-17T10:19:53.421005Z 2022-01-17T10:19:27.508909Z 2022-01-17T10:19:27.471627Z 2022-01-17T10:19:27.417239Z 2022-01-17T10:19:27.358206Z 2022-01-17T10:19:27.320007Z 2022-01-17T10:19:27.281327Z 2022-01-17T10:19:27.242773Z 2022-01-17T10:19:27.201680Z 2022-01-17T10:19:27.147178Z 2022-01-17T10:19:27.098529Z 2022-01-17T10:19:27.050378Z 2022-01-17T10:19:26.978439Z 2022-01-17T10:19:26.881492Z 2022-01-17T10:19:26.774387Z 2022-01-17T10:19:26.715171Z 2022-01-17T10:19:26.668323Z 2022-01-17T10:19:26.633588Z 2022-01-17T10:19:26.599631Z 2022-01-17T10:19:26.564962Z 2022-01-17T10:19:26.532803Z 2022-01-17T10:19:26.498844Z 2022-01-17T10:19:26.462804Z 2022-01-17T10:19:26.425547Z 2022-01-17T10:19:26.391538Z 2022-01-17T10:19:26.356815Z 2022-01-17T10:19:26.324869Z 2022-01-17T10:19:26.288853Z 2022-01-17T10:19:26.251707Z 2022-01-17T10:19:26.216641Z 2022-01-17T10:19:26.182561Z 2022-01-17T10:19:26.148437Z 2022-01-17T10:19:26.112914Z 2022-01-17T10:19:26.076291Z 2022-01-17T10:19:26.039257Z 2022-01-17T10:19:25.996033Z 2022-01-17T10:19:25.954742Z 2022-01-17T10:19:25.916579Z 2022-01-17T10:19:25.873593Z 2022-01-17T10:19:25.823283Z 2022-01-17T10:19:25.774941Z 2022-01-17T10:19:25.720092Z 2022-01-17T10:19:25.679024Z 2022-01-17T10:19:25.640966Z 2022-01-17T10:19:25.591313Z 2022-01-17T10:19:25.515249Z 2022-01-17T10:19:25.439166Z 2022-01-17T10:19:25.379124Z 2022-01-17T10:19:25.291463Z 2022-01-17T10:19:25.222586Z 2022-01-17T10:19:25.135448Z 2022-01-17T10:19:25.079028Z 2022-01-17T10:19:24.966540Z 2022-01-17T10:19:24.869763Z 2022-01-17T10:19:24.775926Z 2022-01-17T10:19:24.638939Z 2022-01-17T10:19:15.743909Z 2022-01-17T10:19:24.453497Z 2022-01-17T10:19:24.306897Z 2022-01-17T10:19:15.410169Z 2022-01-17T10:19:15.098679Z 2022-01-17T10:19:14.780434Z 2022-01-17T10:19:14.500004Z 2022-01-17T10:19:14.352418Z 2022-01-17T10:19:14.080408Z 2022-01-17T10:19:13.800057Z 2022-01-17T10:19:13.527375Z 2022-01-17T10:19:13.301004Z 2022-01-17T10:19:13.215255Z 2022-01-17T10:19:11.731576Z 2022-01-17T10:19:11.665279Z 2022-01-17T10:19:11.615911Z 2022-01-17T10:23:59.907693Z 2022-01-17T10:23:59.803872Z 2022-01-17T10:23:59.677036Z 2022-01-17T10:23:27.255594Z 2022-01-17T10:23:27.214358Z 2022-01-17T10:23:27.171265Z 2022-01-17T10:23:27.113311Z 2022-01-17T10:23:27.074991Z 2022-01-17T10:23:27.032242Z 2022-01-17T10:23:26.989602Z 2022-01-17T10:23:26.952346Z 2022-01-17T10:23:26.909729Z 2022-01-17T10:23:26.869002Z 2022-01-17T10:23:26.831974Z 2022-01-17T10:23:26.796074Z 2022-01-17T10:23:26.758811Z 2022-01-17T10:23:26.718313Z 2022-01-17T10:23:26.675116Z 2022-01-17T10:23:26.637237Z 2022-01-17T10:23:26.587573Z 2022-01-17T10:23:26.545316Z 2022-01-17T10:23:26.497203Z 2022-01-17T10:23:26.443537Z 2022-01-17T10:23:26.349654Z 2022-01-17T10:23:26.283029Z 2022-01-17T10:23:26.203374Z 2022-01-17T10:23:26.142279Z 2022-01-17T10:23:26.074104Z 2022-01-17T10:23:26.040769Z 2022-01-17T10:23:25.969625Z 2022-01-17T10:23:26.007055Z 2022-01-17T10:23:25.934090Z 2022-01-17T10:23:25.898074Z 2022-01-17T10:23:25.860412Z 2022-01-17T10:23:25.826382Z 2022-01-17T10:23:25.715307Z 2022-01-17T10:23:25.642022Z 2022-01-17T10:23:25.591478Z 2022-01-17T10:23:25.497648Z 2022-01-17T10:23:25.368872Z 2022-01-17T10:23:25.261361Z 2022-01-17T10:23:25.166357Z 2022-01-17T10:23:25.075245Z 2022-01-17T10:23:25.008614Z 2022-01-17T10:23:24.944285Z 2022-01-17T10:23:24.882456Z 2022-01-17T10:23:24.820872Z 2022-01-17T10:23:24.754999Z 2022-01-17T10:23:24.712218Z 2022-01-17T10:23:24.660640Z 2022-01-17T10:23:24.616050Z 2022-01-17T10:23:24.548979Z 2022-01-17T10:23:24.498544Z 2022-01-17T10:23:24.453757Z 2022-01-17T10:23:24.404412Z 2022-01-17T10:23:24.284050Z 2022-01-17T10:23:24.199659Z 2022-01-17T10:23:24.141629Z 2022-01-17T10:23:24.104418Z 2022-01-17T10:23:24.065916Z 2022-01-17T10:23:24.031010Z 2022-01-17T10:23:23.985401Z 2022-01-17T10:23:23.890873Z 2022-01-17T10:23:23.941357Z 2022-01-17T10:23:23.846336Z 2022-01-17T10:23:23.803528Z 2022-01-17T10:23:23.762709Z 2022-01-17T10:23:23.724081Z 2022-01-17T10:23:23.670889Z 2022-01-17T10:23:23.622240Z 2022-01-17T10:23:23.580990Z 2022-01-17T10:23:23.537391Z 2022-01-17T10:23:23.474395Z 2022-01-17T10:23:23.430400Z 2022-01-17T10:23:23.390858Z 2022-01-17T10:23:23.341690Z 2022-01-17T10:23:23.284779Z 2022-01-17T10:23:23.236200Z 2022-01-17T10:23:23.195084Z 2022-01-17T10:23:23.150171Z 2022-01-17T10:23:23.112691Z 2022-01-17T10:23:23.020010Z 2022-01-17T10:23:22.946909Z 2022-01-17T10:23:14.026067Z 2022-01-17T10:23:13.887290Z 2022-01-17T10:23:13.698179Z 2022-01-17T10:23:13.600915Z 2022-01-17T10:23:13.449298Z 2022-01-17T10:23:13.240932Z 2022-01-17T10:23:12.957592Z 2022-01-17T10:23:12.804578Z 2022-01-17T10:23:12.678970Z 2022-01-17T10:23:12.588553Z 2022-01-17T10:23:12.398199Z 2022-01-17T10:23:12.250874Z 2022-01-17T10:23:11.996115Z 2022-01-17T10:23:11.771076Z 2022-01-17T10:23:10.045429Z 2022-01-17T10:23:09.873716Z 2022-01-17T10:23:09.278453Z 2022-01-17T10:28:04.684857Z 2022-01-17T10:28:04.619737Z 2022-01-17T10:28:04.563043Z 2022-01-17T10:28:04.494817Z 2022-01-17T10:28:04.380769Z 2022-01-17T10:28:04.242908Z 2022-01-17T10:28:04.128214Z 2022-01-17T10:28:04.072049Z 2022-01-17T10:28:04.016047Z 2022-01-17T10:28:03.954650Z 2022-01-17T10:28:03.868595Z 2022-01-17T10:28:03.789290Z 2022-01-17T10:28:03.722964Z 2022-01-17T10:28:03.655626Z 2022-01-17T10:28:03.599698Z 2022-01-17T10:28:03.534585Z 2022-01-17T10:28:03.472552Z 2022-01-17T10:28:03.399664Z 2022-01-17T10:28:03.328876Z 2022-01-17T10:28:03.252175Z 2022-01-17T10:28:03.195823Z 2022-01-17T10:28:03.077851Z 2022-01-17T10:28:03.126845Z 2022-01-17T10:28:02.996979Z 2022-01-17T10:28:02.840738Z 2022-01-17T10:28:02.695754Z 2022-01-17T10:28:02.519303Z 2022-01-17T10:28:02.398133Z 2022-01-17T10:27:41.086539Z 2022-01-17T10:28:02.252777Z 2022-01-17T10:27:40.965491Z 2022-01-17T10:28:02.134712Z 2022-01-17T10:27:40.794057Z 2022-01-17T10:27:40.698312Z 2022-01-17T10:27:40.640965Z 2022-01-17T10:27:40.590985Z 2022-01-17T10:27:40.466367Z 2022-01-17T10:28:02.034951Z 2022-01-17T10:27:40.357454Z 2022-01-17T10:27:40.268027Z 2022-01-17T10:27:40.203122Z 2022-01-17T10:27:40.169871Z 2022-01-17T10:27:40.136275Z 2022-01-17T10:27:40.103609Z 2022-01-17T10:27:40.067321Z 2022-01-17T10:27:40.014215Z 2022-01-17T10:27:39.912950Z 2022-01-17T10:27:39.815354Z 2022-01-17T10:27:39.707340Z 2022-01-17T10:27:39.617183Z 2022-01-17T10:27:39.563205Z 2022-01-17T10:27:39.524033Z 2022-01-17T10:27:39.449300Z 2022-01-17T10:27:39.489510Z 2022-01-17T10:27:39.404618Z 2022-01-17T10:27:39.348467Z 2022-01-17T10:27:39.309233Z 2022-01-17T10:27:39.274263Z 2022-01-17T10:27:39.237417Z 2022-01-17T10:27:39.192654Z 2022-01-17T10:27:39.144244Z 2022-01-17T10:27:39.107927Z 2022-01-17T10:27:39.071323Z 2022-01-17T10:27:39.027680Z 2022-01-17T10:27:38.985045Z 2022-01-17T10:27:38.917614Z 2022-01-17T10:27:38.795608Z 2022-01-17T10:27:38.711715Z 2022-01-17T10:27:38.617933Z 2022-01-17T10:27:38.335771Z 2022-01-17T10:27:38.476397Z 2022-01-17T10:27:38.229249Z 2022-01-17T10:27:38.091925Z 2022-01-17T10:27:37.979369Z 2022-01-17T10:27:37.915668Z 2022-01-17T10:27:37.823406Z 2022-01-17T10:27:37.770032Z 2022-01-17T10:27:37.715942Z 2022-01-17T10:27:37.657328Z 2022-01-17T10:27:37.598816Z 2022-01-17T10:27:37.442289Z 2022-01-17T10:27:37.199784Z 2022-01-17T10:27:37.066649Z 2022-01-17T10:27:36.897940Z 2022-01-17T10:27:36.692474Z 2022-01-17T10:27:36.484517Z 2022-01-17T10:27:36.315685Z 2022-01-17T10:27:28.610816Z 2022-01-17T10:27:28.458408Z 2022-01-17T10:27:28.279004Z 2022-01-17T10:27:28.133091Z 2022-01-17T10:27:27.969755Z 2022-01-17T10:27:27.817398Z 2022-01-17T10:27:27.659608Z 2022-01-17T10:27:27.471450Z 2022-01-17T10:27:27.308845Z 2022-01-17T10:27:25.570969Z 2022-01-17T10:27:25.535582Z 2022-01-17T10:27:25.481673Z 2022-01-17T10:27:25.446875Z", + "modified_diff": 46.7584736, + "started": "2022-01-17T10:10:29.134586Z 2022-01-17T10:10:27.183323Z 2022-01-17T10:10:28.808463Z 2022-01-17T10:10:26.855565Z 2022-01-17T10:10:28.247999Z 2022-01-17T10:10:26.496410Z 2022-01-17T10:10:27.635648Z 2022-01-17T10:10:26.355648Z 2022-01-17T10:10:27.450393Z 2022-01-17T10:10:26.135995Z 2022-01-17T10:10:26.098197Z 2022-01-17T10:10:25.896455Z 2022-01-17T10:10:26.054625Z 2022-01-17T10:10:25.752754Z 2022-01-17T10:10:25.672530Z 2022-01-17T10:10:25.382837Z 2022-01-17T10:10:25.908044Z 2022-01-17T10:10:25.005049Z 2022-01-17T10:10:25.241634Z 2022-01-17T10:10:24.749180Z 2022-01-17T10:10:24.553779Z 2022-01-17T10:10:20.098437Z 2022-01-17T10:10:19.446079Z 2022-01-17T10:10:19.028862Z 2022-01-17T10:10:18.769327Z 2022-01-17T10:10:18.358621Z 2022-01-17T10:10:17.957245Z 2022-01-17T10:10:17.878767Z 2022-01-17T10:10:16.113696Z 2022-01-17T10:10:15.686963Z 2022-01-17T10:10:15.512913Z 2022-01-17T10:10:15.062916Z 2022-01-17T10:10:14.921954Z 2022-01-17T10:10:14.359722Z 2022-01-17T10:10:14.152988Z 2022-01-17T10:11:10.975086Z 2022-01-17T10:11:10.827415Z 2022-01-17T10:11:10.315428Z 2022-01-17T10:11:10.137675Z 2022-01-17T10:11:10.049979Z 2022-01-17T10:11:09.917481Z 2022-01-17T10:11:09.252304Z 2022-01-17T10:11:09.007038Z 2022-01-17T10:11:08.459257Z 2022-01-17T10:11:08.055502Z 2022-01-17T10:11:07.551879Z 2022-01-17T10:11:07.126518Z 2022-01-17T10:11:06.896317Z 2022-01-17T10:11:06.500807Z 2022-01-17T10:11:02.839548Z 2022-01-17T10:11:06.219797Z 2022-01-17T10:11:02.430034Z 2022-01-17T10:11:02.156329Z 2022-01-17T10:11:01.523037Z 2022-01-17T10:11:01.279095Z 2022-01-17T10:11:00.374153Z 2022-01-17T10:11:00.339442Z 2022-01-17T10:10:59.432732Z 2022-01-17T10:10:59.396195Z 2022-01-17T10:10:58.366332Z 2022-01-17T10:10:57.984059Z 2022-01-17T10:10:57.826895Z 2022-01-17T10:10:57.414477Z 2022-01-17T10:10:56.733896Z 2022-01-17T10:10:56.133346Z 2022-01-17T10:10:55.680601Z 2022-01-17T10:10:55.391791Z 2022-01-17T10:10:54.813757Z 2022-01-17T10:10:54.843770Z 2022-01-17T10:10:54.012879Z 2022-01-17T10:10:53.561624Z 2022-01-17T10:10:53.535064Z 2022-01-17T10:10:53.160997Z 2022-01-17T10:10:52.599432Z 2022-01-17T10:10:52.263927Z 2022-01-17T10:10:51.761592Z 2022-01-17T10:10:51.314460Z 2022-01-17T10:10:51.010587Z 2022-01-17T10:10:50.456759Z 2022-01-17T10:10:50.179504Z 2022-01-17T10:10:49.901749Z 2022-01-17T10:10:49.395491Z 2022-01-17T10:10:49.168404Z 2022-01-17T10:10:49.031705Z 2022-01-17T10:10:48.584067Z 2022-01-17T10:10:48.124206Z 2022-01-17T10:10:47.856268Z 2022-01-17T10:10:47.375596Z 2022-01-17T10:10:47.116460Z 2022-01-17T10:10:46.517326Z 2022-01-17T10:10:46.284212Z 2022-01-17T10:10:45.775416Z 2022-01-17T10:10:45.743536Z 2022-01-17T10:10:45.112910Z 2022-01-17T10:10:44.918256Z 2022-01-17T10:10:44.592107Z 2022-01-17T10:10:44.494471Z 2022-01-17T10:10:44.154825Z 2022-01-17T10:10:30.102788Z 2022-01-17T10:10:27.704061Z 2022-01-17T10:15:56.740405Z 2022-01-17T10:15:56.316099Z 2022-01-17T10:15:56.300459Z 2022-01-17T10:15:55.351193Z 2022-01-17T10:15:55.217476Z 2022-01-17T10:15:54.648224Z 2022-01-17T10:15:54.316312Z 2022-01-17T10:15:54.106886Z 2022-01-17T10:15:53.539046Z 2022-01-17T10:15:53.281778Z 2022-01-17T10:15:52.878582Z 2022-01-17T10:15:52.583271Z 2022-01-17T10:15:52.350838Z 2022-01-17T10:15:51.863274Z 2022-01-17T10:15:51.477754Z 2022-01-17T10:15:51.388484Z 2022-01-17T10:15:51.182563Z 2022-01-17T10:15:50.738017Z 2022-01-17T10:15:50.592676Z 2022-01-17T10:15:50.391225Z 2022-01-17T10:15:50.195615Z 2022-01-17T10:15:49.830192Z 2022-01-17T10:15:49.759412Z 2022-01-17T10:15:49.531640Z 2022-01-17T10:15:39.081022Z 2022-01-17T10:15:38.776334Z 2022-01-17T10:15:38.463444Z 2022-01-17T10:15:38.424530Z 2022-01-17T10:15:38.032998Z 2022-01-17T10:15:37.532570Z 2022-01-17T10:15:37.219229Z 2022-01-17T10:15:36.924203Z 2022-01-17T10:15:36.397659Z 2022-01-17T10:15:36.512299Z 2022-01-17T10:15:35.720718Z 2022-01-17T10:15:35.547822Z 2022-01-17T10:15:34.968406Z 2022-01-17T10:15:34.681704Z 2022-01-17T10:15:34.150603Z 2022-01-17T10:15:34.315411Z 2022-01-17T10:15:33.468612Z 2022-01-17T10:15:33.197523Z 2022-01-17T10:15:32.947136Z 2022-01-17T10:15:32.626585Z 2022-01-17T10:15:32.286796Z 2022-01-17T10:15:32.221109Z 2022-01-17T10:15:31.506382Z 2022-01-17T10:15:31.461461Z 2022-01-17T10:15:30.463971Z 2022-01-17T10:15:30.380088Z 2022-01-17T10:15:29.994722Z 2022-01-17T10:15:30.129940Z 2022-01-17T10:15:29.304561Z 2022-01-17T10:15:29.203852Z 2022-01-17T10:15:28.917647Z 2022-01-17T10:15:28.688027Z 2022-01-17T10:15:28.377262Z 2022-01-17T10:15:28.006908Z 2022-01-17T10:15:27.768672Z 2022-01-17T10:15:27.809489Z 2022-01-17T10:15:27.245637Z 2022-01-17T10:15:26.962816Z 2022-01-17T10:15:26.611115Z 2022-01-17T10:15:26.439008Z 2022-01-17T10:15:25.498640Z 2022-01-17T10:15:25.446299Z 2022-01-17T10:15:24.946574Z 2022-01-17T10:15:24.744106Z 2022-01-17T10:15:24.438040Z 2022-01-17T10:15:24.147373Z 2022-01-17T10:15:23.665815Z 2022-01-17T10:15:23.559925Z 2022-01-17T10:15:23.161754Z 2022-01-17T10:15:22.941776Z 2022-01-17T10:15:22.650022Z 2022-01-17T10:15:22.597494Z 2022-01-17T10:15:22.258858Z 2022-01-17T10:15:21.966813Z 2022-01-17T10:15:21.684553Z 2022-01-17T10:15:21.470216Z 2022-01-17T10:15:21.167467Z 2022-01-17T10:15:20.998269Z 2022-01-17T10:15:20.794911Z 2022-01-17T10:15:20.786837Z 2022-01-17T10:15:11.920020Z 2022-01-17T10:15:11.329117Z 2022-01-17T10:15:10.305253Z 2022-01-17T10:15:10.019391Z 2022-01-17T10:15:08.379740Z 2022-01-17T10:15:08.099871Z 2022-01-17T10:15:07.634059Z 2022-01-17T10:15:07.707845Z 2022-01-17T10:15:06.393796Z 2022-01-17T10:15:06.245319Z 2022-01-17T10:15:01.444570Z 2022-01-17T10:15:01.274230Z 2022-01-17T10:15:01.106461Z 2022-01-17T10:15:01.049742Z 2022-01-17T10:15:00.356905Z 2022-01-17T10:14:59.718431Z 2022-01-17T10:19:59.386706Z 2022-01-17T10:19:59.272908Z 2022-01-17T10:19:59.183123Z 2022-01-17T10:19:59.017293Z 2022-01-17T10:19:58.393729Z 2022-01-17T10:19:58.848039Z 2022-01-17T10:19:57.836640Z 2022-01-17T10:19:57.573115Z 2022-01-17T10:19:57.601735Z 2022-01-17T10:19:56.928102Z 2022-01-17T10:19:56.698745Z 2022-01-17T10:19:56.436174Z 2022-01-17T10:19:56.212713Z 2022-01-17T10:19:55.959091Z 2022-01-17T10:19:51.298606Z 2022-01-17T10:19:55.742706Z 2022-01-17T10:19:55.400249Z 2022-01-17T10:19:51.120505Z 2022-01-17T10:19:55.415704Z 2022-01-17T10:19:50.500502Z 2022-01-17T10:19:50.347887Z 2022-01-17T10:19:50.031614Z 2022-01-17T10:19:49.695307Z 2022-01-17T10:19:49.443517Z 2022-01-17T10:19:49.298106Z 2022-01-17T10:19:48.812716Z 2022-01-17T10:19:48.372272Z 2022-01-17T10:19:48.045785Z 2022-01-17T10:19:55.056825Z 2022-01-17T10:19:47.877574Z 2022-01-17T10:19:47.329640Z 2022-01-17T10:19:46.867853Z 2022-01-17T10:19:46.645748Z 2022-01-17T10:19:46.257012Z 2022-01-17T10:19:45.797948Z 2022-01-17T10:19:45.708615Z 2022-01-17T10:19:45.233186Z 2022-01-17T10:19:44.833968Z 2022-01-17T10:19:44.313606Z 2022-01-17T10:19:43.992982Z 2022-01-17T10:19:43.649435Z 2022-01-17T10:19:43.262232Z 2022-01-17T10:19:42.688608Z 2022-01-17T10:19:42.089211Z 2022-01-17T10:19:41.659155Z 2022-01-17T10:19:41.362767Z 2022-01-17T10:19:41.024840Z 2022-01-17T10:19:40.992338Z 2022-01-17T10:19:40.319638Z 2022-01-17T10:19:40.604091Z 2022-01-17T10:19:39.785867Z 2022-01-17T10:19:39.495214Z 2022-01-17T10:19:39.234514Z 2022-01-17T10:19:38.785326Z 2022-01-17T10:19:38.482415Z 2022-01-17T10:19:38.379306Z 2022-01-17T10:19:38.155713Z 2022-01-17T10:19:37.784225Z 2022-01-17T10:19:37.291703Z 2022-01-17T10:19:37.455730Z 2022-01-17T10:19:36.565016Z 2022-01-17T10:19:36.436720Z 2022-01-17T10:19:35.532501Z 2022-01-17T10:19:35.073235Z 2022-01-17T10:19:34.498500Z 2022-01-17T10:19:34.441727Z 2022-01-17T10:19:33.752884Z 2022-01-17T10:19:33.490660Z 2022-01-17T10:19:33.156907Z 2022-01-17T10:19:33.060018Z 2022-01-17T10:19:32.755409Z 2022-01-17T10:19:32.624265Z 2022-01-17T10:19:32.319763Z 2022-01-17T10:19:31.810878Z 2022-01-17T10:19:31.353554Z 2022-01-17T10:19:30.968011Z 2022-01-17T10:19:30.810898Z 2022-01-17T10:19:30.276001Z 2022-01-17T10:19:29.990881Z 2022-01-17T10:19:29.630202Z 2022-01-17T10:19:29.332970Z 2022-01-17T10:19:28.976318Z 2022-01-17T10:19:28.966052Z 2022-01-17T10:19:28.610357Z 2022-01-17T10:19:20.058897Z 2022-01-17T10:19:28.633940Z 2022-01-17T10:19:28.403215Z 2022-01-17T10:19:20.428646Z 2022-01-17T10:19:19.511706Z 2022-01-17T10:19:18.943491Z 2022-01-17T10:19:18.864452Z 2022-01-17T10:19:18.298223Z 2022-01-17T10:19:17.706913Z 2022-01-17T10:19:17.038689Z 2022-01-17T10:19:16.899965Z 2022-01-17T10:19:16.505941Z 2022-01-17T10:19:16.610349Z 2022-01-17T10:19:12.538016Z 2022-01-17T10:19:12.312983Z 2022-01-17T10:19:12.067559Z 2022-01-17T10:24:01.012973Z 2022-01-17T10:24:00.744169Z 2022-01-17T10:24:00.618914Z 2022-01-17T10:23:55.971546Z 2022-01-17T10:23:55.761080Z 2022-01-17T10:23:55.393471Z 2022-01-17T10:23:54.778615Z 2022-01-17T10:23:54.403392Z 2022-01-17T10:23:54.095369Z 2022-01-17T10:23:53.804461Z 2022-01-17T10:23:53.727765Z 2022-01-17T10:23:53.251640Z 2022-01-17T10:23:53.039812Z 2022-01-17T10:23:52.871590Z 2022-01-17T10:23:52.512600Z 2022-01-17T10:23:52.267188Z 2022-01-17T10:23:51.776687Z 2022-01-17T10:23:51.541804Z 2022-01-17T10:23:50.997613Z 2022-01-17T10:23:50.543606Z 2022-01-17T10:23:49.962633Z 2022-01-17T10:23:49.836178Z 2022-01-17T10:23:49.434356Z 2022-01-17T10:23:49.138174Z 2022-01-17T10:23:48.845715Z 2022-01-17T10:23:48.388575Z 2022-01-17T10:23:48.083980Z 2022-01-17T10:23:47.728327Z 2022-01-17T10:23:47.400620Z 2022-01-17T10:23:46.717740Z 2022-01-17T10:23:47.041301Z 2022-01-17T10:23:46.388246Z 2022-01-17T10:23:45.750679Z 2022-01-17T10:23:45.504567Z 2022-01-17T10:23:44.996644Z 2022-01-17T10:23:44.602749Z 2022-01-17T10:23:43.882686Z 2022-01-17T10:23:43.563648Z 2022-01-17T10:23:42.555313Z 2022-01-17T10:23:42.075402Z 2022-01-17T10:23:41.761495Z 2022-01-17T10:23:41.980804Z 2022-01-17T10:23:41.280598Z 2022-01-17T10:23:41.580571Z 2022-01-17T10:23:40.844294Z 2022-01-17T10:23:40.479221Z 2022-01-17T10:23:39.969643Z 2022-01-17T10:23:39.804213Z 2022-01-17T10:23:38.759894Z 2022-01-17T10:23:38.613559Z 2022-01-17T10:23:38.261624Z 2022-01-17T10:23:38.023893Z 2022-01-17T10:23:37.444638Z 2022-01-17T10:23:37.204165Z 2022-01-17T10:23:36.569883Z 2022-01-17T10:23:36.638711Z 2022-01-17T10:23:35.887395Z 2022-01-17T10:23:35.941863Z 2022-01-17T10:23:35.365791Z 2022-01-17T10:23:35.237298Z 2022-01-17T10:23:34.747116Z 2022-01-17T10:23:34.373275Z 2022-01-17T10:23:33.206278Z 2022-01-17T10:23:33.454456Z 2022-01-17T10:23:32.737159Z 2022-01-17T10:23:32.338322Z 2022-01-17T10:23:32.036401Z 2022-01-17T10:23:31.625339Z 2022-01-17T10:23:31.062227Z 2022-01-17T10:23:31.000765Z 2022-01-17T10:23:30.663852Z 2022-01-17T10:23:30.332248Z 2022-01-17T10:23:29.867749Z 2022-01-17T10:23:29.726735Z 2022-01-17T10:23:29.370276Z 2022-01-17T10:23:29.217677Z 2022-01-17T10:23:28.795737Z 2022-01-17T10:23:28.575448Z 2022-01-17T10:23:28.280607Z 2022-01-17T10:23:28.270783Z 2022-01-17T10:23:28.044554Z 2022-01-17T10:23:27.740506Z 2022-01-17T10:23:27.725310Z 2022-01-17T10:23:19.972382Z 2022-01-17T10:23:19.422488Z 2022-01-17T10:23:19.607043Z 2022-01-17T10:23:18.765672Z 2022-01-17T10:23:19.430706Z 2022-01-17T10:23:18.211682Z 2022-01-17T10:23:18.432748Z 2022-01-17T10:23:17.792158Z 2022-01-17T10:23:17.611103Z 2022-01-17T10:23:16.995368Z 2022-01-17T10:23:16.251601Z 2022-01-17T10:23:15.985177Z 2022-01-17T10:23:15.166770Z 2022-01-17T10:23:15.202104Z 2022-01-17T10:23:10.987181Z 2022-01-17T10:23:10.457422Z 2022-01-17T10:23:09.599560Z 2022-01-17T10:28:15.498305Z 2022-01-17T10:28:14.405364Z 2022-01-17T10:28:14.301308Z 2022-01-17T10:28:13.722611Z 2022-01-17T10:28:13.491222Z 2022-01-17T10:28:12.610567Z 2022-01-17T10:28:12.450352Z 2022-01-17T10:28:11.884836Z 2022-01-17T10:28:11.734265Z 2022-01-17T10:28:11.241618Z 2022-01-17T10:28:10.955185Z 2022-01-17T10:28:10.380197Z 2022-01-17T10:28:09.925065Z 2022-01-17T10:28:09.642020Z 2022-01-17T10:28:09.301704Z 2022-01-17T10:28:09.119888Z 2022-01-17T10:28:08.869286Z 2022-01-17T10:28:08.554183Z 2022-01-17T10:28:08.238185Z 2022-01-17T10:28:08.287953Z 2022-01-17T10:28:07.583119Z 2022-01-17T10:28:07.216303Z 2022-01-17T10:28:07.418397Z 2022-01-17T10:28:06.838512Z 2022-01-17T10:28:06.666307Z 2022-01-17T10:28:06.326626Z 2022-01-17T10:28:06.202692Z 2022-01-17T10:28:06.079683Z 2022-01-17T10:27:59.890307Z 2022-01-17T10:28:05.634008Z 2022-01-17T10:27:59.606657Z 2022-01-17T10:28:05.317333Z 2022-01-17T10:27:59.248123Z 2022-01-17T10:27:58.902737Z 2022-01-17T10:27:58.543393Z 2022-01-17T10:27:58.325638Z 2022-01-17T10:27:58.176308Z 2022-01-17T10:28:05.148984Z 2022-01-17T10:27:57.847914Z 2022-01-17T10:27:57.914724Z 2022-01-17T10:27:57.251453Z 2022-01-17T10:27:56.939503Z 2022-01-17T10:27:56.516211Z 2022-01-17T10:27:56.072205Z 2022-01-17T10:27:55.745461Z 2022-01-17T10:27:55.491698Z 2022-01-17T10:27:55.027990Z 2022-01-17T10:27:54.753287Z 2022-01-17T10:27:54.073805Z 2022-01-17T10:27:53.690911Z 2022-01-17T10:27:53.281241Z 2022-01-17T10:27:53.138062Z 2022-01-17T10:27:52.530056Z 2022-01-17T10:27:52.707575Z 2022-01-17T10:27:51.822407Z 2022-01-17T10:27:51.638556Z 2022-01-17T10:27:50.830376Z 2022-01-17T10:27:50.676827Z 2022-01-17T10:27:50.252824Z 2022-01-17T10:27:49.795041Z 2022-01-17T10:27:49.402503Z 2022-01-17T10:27:49.328688Z 2022-01-17T10:27:48.554612Z 2022-01-17T10:27:48.152481Z 2022-01-17T10:27:47.728559Z 2022-01-17T10:27:47.498535Z 2022-01-17T10:27:46.922951Z 2022-01-17T10:27:47.103905Z 2022-01-17T10:27:46.236387Z 2022-01-17T10:27:45.676075Z 2022-01-17T10:27:46.597293Z 2022-01-17T10:27:45.513492Z 2022-01-17T10:27:45.119638Z 2022-01-17T10:27:44.858827Z 2022-01-17T10:27:44.634428Z 2022-01-17T10:27:44.567213Z 2022-01-17T10:27:44.051204Z 2022-01-17T10:27:43.788098Z 2022-01-17T10:27:43.273334Z 2022-01-17T10:27:43.219543Z 2022-01-17T10:27:42.895147Z 2022-01-17T10:27:42.746567Z 2022-01-17T10:27:42.531485Z 2022-01-17T10:27:42.265548Z 2022-01-17T10:27:41.975381Z 2022-01-17T10:27:41.695217Z 2022-01-17T10:27:41.554296Z 2022-01-17T10:27:32.530112Z 2022-01-17T10:27:32.766063Z 2022-01-17T10:27:32.185271Z 2022-01-17T10:27:31.943983Z 2022-01-17T10:27:30.870242Z 2022-01-17T10:27:30.631885Z 2022-01-17T10:27:30.610984Z 2022-01-17T10:27:29.700612Z 2022-01-17T10:27:29.258698Z 2022-01-17T10:27:26.528913Z 2022-01-17T10:27:26.149537Z 2022-01-17T10:27:25.907193Z 2022-01-17T10:27:25.808338Z", + "started_diff": 52.453319799999996, + "successful_job_ids": " [99, 98, 96, 95, 94, 93, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 75, 74, 73, 72, 71, 70, 69, 67, 66, 65, 63, 62, 61, 60, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 157, 156, 155, 154, 153, 150, 149, 146, 145, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 107, 106, 105, 104, 103, 101, 100, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 173, 172, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 273, 272, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, 472]" + }, + "started": "2022-01-17T10:53:57.769709+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-api.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-api.json new file mode 100644 index 00000000..570c36e6 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-api.json @@ -0,0 +1,1244 @@ +{ + "ended": "2022-01-17T23:14:26.331100+00:00", + "id": "run-2022-01-17T20_14_07_784_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 2201.3333333333335, + "max": 24126.4, + "mean": 5298.565340777629, + "median": 3328.666666666667, + "min": 1316.2, + "non_zero_mean": 5298.565340777629, + "non_zero_median": 3328.666666666667, + "percentile25": 2798.6, + "percentile75": 4999.933333333333, + "percentile90": 12798.000000000002, + "percentile99": 22790.389333333333, + "percentile999": 23914.56426666671, + "range": 22810.2, + "samples": 263, + "stdev": 4787.704454544304, + "sum": 1393522.6846245178 + } + }, + "cpu": { + "system": { + "iqr": 0.18199999999999933, + "max": 1.7620000000000042, + "mean": 0.2731609821234239, + "median": 0.1146666666666647, + "min": 0.033333333333329544, + "non_zero_mean": 0.2731609821234239, + "non_zero_median": 0.1146666666666647, + "percentile25": 0.061999999999998556, + "percentile75": 0.24399999999999789, + "percentile90": 0.8737333333333384, + "percentile99": 1.7153333333333323, + "percentile999": 1.7574586666666712, + "range": 1.7286666666666748, + "samples": 263, + "stdev": 0.39858568463576555, + "sum": 71.8413382984605 + }, + "user": { + "iqr": 0.5056666666666464, + "max": 6.234666666666651, + "mean": 1.3628395282067407, + "median": 0.7660000000000006, + "min": 0.3826666666666672, + "non_zero_mean": 1.3628395282067407, + "non_zero_median": 0.7660000000000006, + "percentile25": 0.5999999999999924, + "percentile75": 1.1056666666666388, + "percentile90": 3.753600000000038, + "percentile99": 6.130240000000009, + "percentile999": 6.217549333333313, + "range": 5.851999999999983, + "samples": 263, + "stdev": 1.4736618463015687, + "sum": 358.4267959183729 + } + }, + "entropy_available_bits": { + "iqr": 1.9333333333333331, + "max": 388.6, + "mean": 20.09201705685437, + "median": 1.7999999999999998, + "min": 0.2, + "non_zero_mean": 20.09201705685437, + "non_zero_median": 1.7999999999999998, + "percentile25": 0.6, + "percentile75": 2.533333333333333, + "percentile90": 4.253333333333334, + "percentile99": 381.11466666666666, + "percentile999": 387.7790666666669, + "range": 388.40000000000003, + "samples": 263, + "stdev": 80.4587764978576, + "sum": 5284.2004859527015 + }, + "memory": { + "used": { + "iqr": 263528448.0, + "max": 14549192704.0, + "mean": 6487833782.996198, + "median": 6060511232.0, + "min": 5783609344.0, + "non_zero_mean": 6487833782.996198, + "non_zero_median": 6060511232.0, + "percentile25": 5984696320.0, + "percentile75": 6248224768.0, + "percentile90": 7200993280.0, + "percentile99": 12705514946.559998, + "percentile999": 14471343038.464016, + "range": 8765583360.0, + "samples": 263, + "stdev": 1383015405.975631, + "sum": 1706300284928.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 17.266666666666666, + "mean": 0.09226869455006337, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 4.044444444444444, + "non_zero_median": 1.5333333333333332, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 1.4853333333333314, + "percentile999": 13.458933333334151, + "range": 17.266666666666666, + "samples": 263, + "stdev": 1.085787690902772, + "sum": 24.26666666666667 + } + }, + "writes_completed": { + "total": { + "iqr": 85.43333333333334, + "max": 1579.5333333333335, + "mean": 113.72748828586957, + "median": 29.13333333333333, + "min": 0.0, + "non_zero_mean": 120.12180489632007, + "non_zero_median": 32.53333333333333, + "percentile25": 1.3666666666666667, + "percentile75": 86.8, + "percentile90": 341.7066666666667, + "percentile99": 1049.952, + "percentile999": 1477.2310666666888, + "range": 1579.5333333333335, + "samples": 263, + "stdev": 235.79796483469642, + "sum": 29910.329419183705 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 912.8666666666668, + "max": 3114.4666666666667, + "mean": 1623.7249683143218, + "median": 1516.0, + "min": 478.73333333333335, + "non_zero_mean": 1623.7249683143218, + "non_zero_median": 1516.0, + "percentile25": 1187.4666666666667, + "percentile75": 2100.3333333333335, + "percentile90": 2397.9066666666668, + "percentile99": 2915.6919999999996, + "percentile999": 3083.9524000000065, + "range": 2635.7333333333336, + "samples": 263, + "stdev": 578.3634981809864, + "sum": 427039.66666666686 + } + }, + "cpu": { + "system": { + "iqr": 0.0456666666666687, + "max": 0.2726666666666669, + "mean": 0.087191381495564, + "median": 0.07533333333333492, + "min": 0.01533333333333265, + "non_zero_mean": 0.087191381495564, + "non_zero_median": 0.07533333333333492, + "percentile25": 0.05733333333333235, + "percentile75": 0.10300000000000105, + "percentile90": 0.15733333333333385, + "percentile99": 0.23380000000000054, + "percentile999": 0.26585466666666857, + "range": 0.25733333333333425, + "samples": 263, + "stdev": 0.04593612255582003, + "sum": 22.931333333333313 + }, + "user": { + "iqr": 0.12133333333333288, + "max": 0.8013333333333398, + "mean": 0.20493536121673003, + "median": 0.1953333333333357, + "min": 0.018666666666668635, + "non_zero_mean": 0.20493536121673003, + "non_zero_median": 0.1953333333333357, + "percentile25": 0.1330000000000022, + "percentile75": 0.2543333333333351, + "percentile90": 0.3242666666666628, + "percentile99": 0.5434800000000012, + "percentile999": 0.7898053333333388, + "range": 0.7826666666666712, + "samples": 263, + "stdev": 0.10872406169033462, + "sum": 53.89799999999999 + } + }, + "entropy_available_bits": { + "iqr": 1.4666666666666666, + "max": 211.13333333333333, + "mean": 11.246894803548797, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 20.83051643192488, + "non_zero_median": 1.4666666666666666, + "percentile25": 0.0, + "percentile75": 1.4666666666666666, + "percentile90": 3.053333333333334, + "percentile99": 210.95866666666666, + "percentile999": 211.0984, + "range": 211.13333333333333, + "samples": 263, + "stdev": 45.594978531555014, + "sum": 2957.9333333333334 + }, + "memory": { + "used": { + "iqr": 21147648.0, + "max": 1008123904.0, + "mean": 424682048.24334604, + "median": 386408448.0, + "min": 369295360.0, + "non_zero_mean": 424682048.24334604, + "non_zero_median": 386408448.0, + "percentile25": 378808320.0, + "percentile75": 399955968.0, + "percentile90": 495516876.80000013, + "percentile99": 942712176.64, + "percentile999": 1007666741.2480001, + "range": 638828544.0, + "samples": 263, + "stdev": 114396095.08841819, + "sum": 111691378688.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 5734.4, + "max": 59480532.2, + "mean": 1338937.5335868187, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3827614.9057971016, + "non_zero_median": 14472.533333333333, + "percentile25": 0.0, + "percentile75": 5734.4, + "percentile90": 35498.6666666667, + "percentile99": 57718673.29866666, + "percentile999": 59390673.605866686, + "range": 59480532.2, + "samples": 263, + "stdev": 8705182.938919915, + "sum": 352140571.3333332 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 437.095227, + "mean": 2.5922714448669204, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 17.94124710526316, + "non_zero_median": 0.12037999999999999, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.020262800000000022, + "percentile99": 28.74976921999967, + "percentile999": 365.5371775880154, + "range": 437.095227, + "samples": 263, + "stdev": 29.085800687750883, + "sum": 681.7673899999999 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.6432789999999999, + "max": 422.097502, + "mean": 26.59799631178707, + "median": 0.0, + "min": -0.000758, + "non_zero_mean": 65.37638345794393, + "non_zero_median": 0.968944, + "percentile25": 0.0, + "percentile75": 0.6432789999999999, + "percentile90": 91.47024800000004, + "percentile99": 382.79963539999994, + "percentile999": 418.1693192320009, + "range": 422.09826000000004, + "samples": 263, + "stdev": 79.49494902258783, + "sum": 6995.273030000004 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 5.866666666666666, + "mean": 0.17363751584283904, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 3.261904761904762, + "non_zero_median": 3.3, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 4.243999999999998, + "percentile999": 5.552266666666734, + "range": 5.866666666666666, + "samples": 263, + "stdev": 0.7851772950379865, + "sum": 45.666666666666664 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9998.2, + "mean": 460.0045627376426, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8641.514285714286, + "non_zero_median": 9322.233333333334, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9992.165333333334, + "percentile999": 9997.798266666667, + "range": 9998.2, + "samples": 263, + "stdev": 1970.9545960758467, + "sum": 120981.20000000001 + }, + "pg_stat_database_blks_hit": { + "iqr": 20403.300000000003, + "max": 219950.2, + "mean": 25814.713561470217, + "median": 25806.533333333333, + "min": 2587.9333333333334, + "non_zero_mean": 25814.713561470217, + "non_zero_median": 25806.533333333333, + "percentile25": 12773.133333333333, + "percentile75": 33176.433333333334, + "percentile90": 43433.37333333335, + "percentile99": 68368.07333333319, + "percentile999": 209307.84733333564, + "range": 217362.2666666667, + "samples": 263, + "stdev": 21202.4650745242, + "sum": 6789269.666666665 + }, + "pg_stat_database_blks_read": { + "iqr": 0.5666666666666667, + "max": 215.13333333333333, + "mean": 1.4157160963244613, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.773015873015873, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6333333333333333, + "percentile90": 2.1333333333333333, + "percentile99": 4.069333333333331, + "percentile999": 160.60240000001173, + "range": 215.13333333333333, + "samples": 263, + "stdev": 13.264216137960991, + "sum": 372.3333333333329 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 263, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 0.5666666666666667, + "max": 215.13333333333333, + "mean": 1.4157160963244613, + "median": 0.2, + "min": 0.0, + "non_zero_mean": 1.773015873015873, + "non_zero_median": 0.3333333333333333, + "percentile25": 0.06666666666666667, + "percentile75": 0.6333333333333333, + "percentile90": 2.1333333333333333, + "percentile99": 4.069333333333331, + "percentile999": 160.60240000001173, + "range": 215.13333333333333, + "samples": 263, + "stdev": 13.264216137960991, + "sum": 372.3333333333329 + }, + "pg_stat_database_tup_deleted": { + "iqr": 2.066666666666667, + "max": 4005.733333333333, + "mean": 41.26692015209125, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 102.38867924528302, + "non_zero_median": 14.933333333333334, + "percentile25": 0.0, + "percentile75": 2.066666666666667, + "percentile90": 59.34666666666668, + "percentile99": 76.46399999999997, + "percentile999": 4004.6853333333333, + "range": 4005.733333333333, + "samples": 263, + "stdev": 348.24337416595137, + "sum": 10853.199999999997 + }, + "pg_stat_database_tup_fetched": { + "iqr": 10619.066666666668, + "max": 419391.73333333334, + "mean": 14631.13485424588, + "median": 13644.533333333333, + "min": 1401.8666666666666, + "non_zero_mean": 14631.13485424588, + "non_zero_median": 13644.533333333333, + "percentile25": 6538.833333333334, + "percentile75": 17157.9, + "percentile90": 24487.240000000013, + "percentile99": 37418.69733333332, + "percentile999": 320121.12106668804, + "range": 417989.8666666667, + "samples": 263, + "stdev": 26366.179286083225, + "sum": 3847988.4666666645 + }, + "pg_stat_database_tup_inserted": { + "iqr": 9.033333333333333, + "max": 8005.733333333334, + "mean": 46.22915082382763, + "median": 2.533333333333333, + "min": 0.0, + "non_zero_mean": 55.51719939117199, + "non_zero_median": 3.7333333333333334, + "percentile25": 1.1333333333333333, + "percentile75": 10.166666666666666, + "percentile90": 61.56000000000002, + "percentile99": 144.03199999999998, + "percentile999": 5950.745066667108, + "range": 8005.733333333334, + "samples": 263, + "stdev": 493.7880444962637, + "sum": 12158.266666666665 + }, + "pg_stat_database_tup_returned": { + "iqr": 39473.53333333333, + "max": 1276142.7333333334, + "mean": 52181.903168567806, + "median": 41177.666666666664, + "min": 3375.733333333333, + "non_zero_mean": 52181.903168567806, + "non_zero_median": 41177.666666666664, + "percentile25": 21146.966666666667, + "percentile75": 60620.5, + "percentile90": 86690.52, + "percentile99": 165897.97199999995, + "percentile999": 1183922.3314666864, + "range": 1272767.0, + "samples": 263, + "stdev": 98817.99520852728, + "sum": 13723840.533333346 + }, + "pg_stat_database_tup_updated": { + "iqr": 12.633333333333335, + "max": 62.13333333333333, + "mean": 10.383776932826363, + "median": 7.333333333333333, + "min": 0.2, + "non_zero_mean": 10.383776932826363, + "non_zero_median": 7.333333333333333, + "percentile25": 3.6, + "percentile75": 16.233333333333334, + "percentile90": 23.386666666666667, + "percentile99": 43.55333333333329, + "percentile999": 59.792800000000504, + "range": 61.93333333333333, + "samples": 263, + "stdev": 9.757285549974474, + "sum": 2730.9333333333325 + }, + "pg_stat_database_xact_commit": { + "iqr": 118.93333333333335, + "max": 707.8666666666667, + "mean": 122.45652724968315, + "median": 90.2, + "min": 4.133333333333334, + "non_zero_mean": 122.45652724968315, + "non_zero_median": 90.2, + "percentile25": 39.733333333333334, + "percentile75": 158.66666666666669, + "percentile90": 271.2266666666668, + "percentile99": 568.5479999999999, + "percentile999": 699.1857333333352, + "range": 703.7333333333333, + "samples": 263, + "stdev": 124.50811827674849, + "sum": 32206.066666666673 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.6, + "mean": 0.12648922686945502, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.12648922686945502, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.4666666666666667, + "percentile99": 0.5586666666666663, + "percentile999": 0.6, + "range": 0.5333333333333333, + "samples": 263, + "stdev": 0.1456306304452119, + "sum": 33.26666666666667 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 263, + "stdev": 0.0, + "sum": 0.0 + } + }, + "writes_completed": { + "total": { + "iqr": 25.9, + "max": 155.86666666666667, + "mean": 39.72623574144487, + "median": 32.266666666666666, + "min": 2.2666666666666666, + "non_zero_mean": 39.72623574144487, + "non_zero_median": 32.266666666666666, + "percentile25": 23.633333333333333, + "percentile75": 49.53333333333333, + "percentile90": 71.05333333333333, + "percentile99": 115.07466666666666, + "percentile999": 147.4302666666685, + "range": 153.6, + "samples": 263, + "stdev": 23.595738973237385, + "sum": 10448.0 + } + } + } + }, + "name": "API performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "started": "2022-01-17T22:08:39.946676+00:00", + "test_api_benchmark_min_rounds": 10, + "test_api_credentials_per_org": 5, + "test_api_groups_per_host": 5, + "test_api_host_count": 1000, + "test_api_host_groups_per_inv": 5, + "test_api_inventories_per_org": 5, + "test_api_labels_per_jt": 5, + "test_api_notification_templates_per_org": 5, + "test_api_org_count": 50, + "test_api_projects_per_org": 1, + "test_api_teams_per_org": 5, + "test_api_users_per_org": 5 + }, + "result": "FAIL", + "results": { + "benchmark_id_guess": "Linux-CPython-3.8.8-64bit_run-2022-01-17T20_14_07_784_0000", + "ended": "2022-01-17T23:14:21.092607+00:00", + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_org": { + "iqr": 0.02052949524977521, + "iterations": 1, + "max": 0.5694939400000294, + "mean": 0.5490026311332864, + "median": 0.5459972969993032, + "min": 0.5270312410002589, + "rounds": 15, + "stddev": 0.012469867950188118 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_associate_user_with_team": { + "iqr": 0.013974814000221158, + "iterations": 1, + "max": 1.3716056670000398, + "mean": 0.6047011510666683, + "median": 0.5537885049998295, + "min": 0.5238935229999697, + "rounds": 15, + "stddev": 0.21243590299733006 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_cloud_credential_creation": { + "iqr": 0.012667984000472643, + "iterations": 1, + "max": 0.4132383740006844, + "mean": 0.40486270890014564, + "median": 0.40848233650012844, + "min": 0.3924618199998804, + "rounds": 10, + "stddev": 0.007899205939174727 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host": { + "iqr": 0.009598171000106959, + "iterations": 1, + "max": 1.0464230679999673, + "mean": 0.2737346141699618, + "median": 0.2642845059999672, + "min": 0.2510782549998112, + "rounds": 100, + "stddev": 0.07861022798661882 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_create_host_and_associate_host_with_group": { + "iqr": 0.013217404499755503, + "iterations": 1, + "max": 0.5375439889994595, + "mean": 0.4731216229200072, + "median": 0.4711320534997867, + "min": 0.45069552099994326, + "rounds": 100, + "stddev": 0.012506646538151178 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[activity_stream]": { + "iqr": 0.11993104500015761, + "iterations": 1, + "max": 1.43629570199937, + "mean": 1.3504680632999224, + "median": 1.3305401094994522, + "min": 1.282194342999901, + "rounds": 10, + "stddev": 0.05673662279894134 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ad_hoc_commands]": { + "iqr": 0.0017816724996464472, + "iterations": 1, + "max": 0.19400748099997145, + "mean": 0.08048177239991977, + "median": 0.072606218999681, + "min": 0.06966386400017655, + "rounds": 15, + "stddev": 0.031452373811608966 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[applications]": { + "iqr": 0.0014690240004711086, + "iterations": 1, + "max": 0.07419363600001816, + "mean": 0.07113793671416195, + "median": 0.07117912250032532, + "min": 0.06847124799969606, + "rounds": 14, + "stddev": 0.0014297046770708351 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[config]": { + "iqr": 0.01881939099985175, + "iterations": 1, + "max": 0.11570939000012004, + "mean": 0.1035355057999368, + "median": 0.09913681899979565, + "min": 0.09257827100009308, + "rounds": 10, + "stddev": 0.009756189139519015 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credential_types]": { + "iqr": 0.0021627749993058387, + "iterations": 1, + "max": 0.11296740000034333, + "mean": 0.10990787340006136, + "median": 0.10970790700002908, + "min": 0.10710980200019549, + "rounds": 10, + "stddev": 0.0019190937671121466 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[credentials]": { + "iqr": 0.012065006999364414, + "iterations": 1, + "max": 0.397289940000519, + "mean": 0.2854541604000588, + "median": 0.2747740560002967, + "min": 0.25853543900029763, + "rounds": 10, + "stddev": 0.04004414643087356 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[dashboard]": { + "iqr": 0.0047112620013649575, + "iterations": 1, + "max": 0.11403867700028059, + "mean": 0.10967331199981345, + "median": 0.11023981249945791, + "min": 0.10179498700017575, + "rounds": 10, + "stddev": 0.003690234874415585 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[groups]": { + "iqr": 0.0017742200000157027, + "iterations": 1, + "max": 0.07464149099996575, + "mean": 0.0722534950666765, + "median": 0.07175592699968547, + "min": 0.07055449099971156, + "rounds": 15, + "stddev": 0.0012613494691928013 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[hosts]": { + "iqr": 0.02231571199990867, + "iterations": 1, + "max": 0.6513491269997758, + "mean": 0.6252644084000167, + "median": 0.6245186285000273, + "min": 0.5918586130001131, + "rounds": 10, + "stddev": 0.018082807192504886 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instance_groups]": { + "iqr": 0.0038762219999171066, + "iterations": 1, + "max": 0.19926268499966682, + "mean": 0.10304694499989803, + "median": 0.09266608300004009, + "min": 0.09075412600031996, + "rounds": 11, + "stddev": 0.03201967000037446 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[instances]": { + "iqr": 0.0012618069995369297, + "iterations": 1, + "max": 0.20320995600013703, + "mean": 0.11184903390003456, + "median": 0.10191738899993652, + "min": 0.09682100499958324, + "rounds": 10, + "stddev": 0.032158978102756475 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory]": { + "iqr": 0.0031904540010145865, + "iterations": 1, + "max": 0.2854557000000568, + "mean": 0.1873936607997166, + "median": 0.17821875649951835, + "min": 0.164141608999671, + "rounds": 10, + "stddev": 0.03481806196664629 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_sources]": { + "iqr": 0.0018050999997285544, + "iterations": 1, + "max": 0.07712012299998605, + "mean": 0.07511005242849933, + "median": 0.07542281250016458, + "min": 0.07210254100027669, + "rounds": 14, + "stddev": 0.0013692227928483282 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[inventory_updates]": { + "iqr": 0.002910795999923721, + "iterations": 1, + "max": 0.08636511599979713, + "mean": 0.07666990714298143, + "median": 0.07621483500042814, + "min": 0.0717936089995419, + "rounds": 14, + "stddev": 0.0034280702683924004 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[job_templates]": { + "iqr": 0.020957208000254468, + "iterations": 1, + "max": 0.4507414740000968, + "mean": 0.3754196254998533, + "median": 0.37190578149966314, + "min": 0.3449936759998309, + "rounds": 10, + "stddev": 0.02934745485569794 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[jobs]": { + "iqr": 0.012406896000356937, + "iterations": 1, + "max": 0.46579568200013455, + "mean": 0.3847568637997938, + "median": 0.38103698699933375, + "min": 0.3492605499995989, + "rounds": 10, + "stddev": 0.030556530121740207 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[labels]": { + "iqr": 0.007646993999742335, + "iterations": 1, + "max": 0.2072708670002612, + "mean": 0.10778578118184999, + "median": 0.09829667500071082, + "min": 0.09278575699954672, + "rounds": 11, + "stddev": 0.03322941233655962 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[me]": { + "iqr": 0.004915974500363518, + "iterations": 1, + "max": 0.15150652000011178, + "mean": 0.09444173508336462, + "median": 0.08727001750003183, + "min": 0.0837178789997779, + "rounds": 12, + "stddev": 0.01978812460818419 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notification_templates]": { + "iqr": 0.004411928999616066, + "iterations": 1, + "max": 0.18214683500082174, + "mean": 0.17623133600018265, + "median": 0.17792933300006553, + "min": 0.16661020900028234, + "rounds": 10, + "stddev": 0.005007709870321456 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[notifications]": { + "iqr": 0.0044311530009508715, + "iterations": 1, + "max": 0.08571715400012181, + "mean": 0.07821149514282817, + "median": 0.077132884999628, + "min": 0.0736819009998726, + "rounds": 14, + "stddev": 0.0033996076294601366 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[organizations]": { + "iqr": 0.0028608359998543165, + "iterations": 1, + "max": 0.2122723339998629, + "mean": 0.20804926629989495, + "median": 0.20820817300000272, + "min": 0.20319077299973287, + "rounds": 10, + "stddev": 0.0026443147115171478 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[ping]": { + "iqr": 0.0024082385004930984, + "iterations": 1, + "max": 0.08291697300046508, + "mean": 0.07953287015397319, + "median": 0.07947095300005458, + "min": 0.07635544400000072, + "rounds": 13, + "stddev": 0.0018272497911911091 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[project_updates]": { + "iqr": 0.02208429700021952, + "iterations": 1, + "max": 0.3557418590007728, + "mean": 0.33652040640008635, + "median": 0.33374368749991845, + "min": 0.31686594400071044, + "rounds": 10, + "stddev": 0.013018797150106086 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[projects]": { + "iqr": 0.007240307000756729, + "iterations": 1, + "max": 0.2482378790000439, + "mean": 0.2339024038002208, + "median": 0.23386859850052133, + "min": 0.222687610000321, + "rounds": 10, + "stddev": 0.007326656822285616 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[roles]": { + "iqr": 0.009092889000385185, + "iterations": 1, + "max": 0.17263894900042942, + "mean": 0.16581265850018098, + "median": 0.16732773749981789, + "min": 0.15700439900047058, + "rounds": 10, + "stddev": 0.005153518017781408 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[schedules]": { + "iqr": 0.12359061400002247, + "iterations": 1, + "max": 0.23506935400018847, + "mean": 0.16887794330004907, + "median": 0.16832671600013782, + "min": 0.10269303000040964, + "rounds": 10, + "stddev": 0.06581265219392997 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[settings]": { + "iqr": 0.0030575749997296953, + "iterations": 1, + "max": 0.08220335599980899, + "mean": 0.07653267271423049, + "median": 0.0765466764996745, + "min": 0.07154284099942743, + "rounds": 14, + "stddev": 0.002700980887737742 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_job_templates]": { + "iqr": 0.009332167999673402, + "iterations": 1, + "max": 0.21693747000063013, + "mean": 0.11135087259999636, + "median": 0.09829825599990727, + "min": 0.09388800899978378, + "rounds": 10, + "stddev": 0.03774083764715417 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[system_jobs]": { + "iqr": 0.0021869592503662716, + "iterations": 1, + "max": 0.0779446429996824, + "mean": 0.07265976646151383, + "median": 0.0722530220000408, + "min": 0.07067844100038201, + "rounds": 13, + "stddev": 0.001966025282529557 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[teams]": { + "iqr": 0.0035846360005962197, + "iterations": 1, + "max": 0.1434144090007976, + "mean": 0.12480894430009357, + "median": 0.12323355750004339, + "min": 0.12078653600019607, + "rounds": 10, + "stddev": 0.006752841896348278 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[tokens]": { + "iqr": 0.0018282720002389397, + "iterations": 1, + "max": 0.07807648300058645, + "mean": 0.07396295500023241, + "median": 0.07334771399973761, + "min": 0.07158363700000336, + "rounds": 14, + "stddev": 0.0019885047645148562 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_job_templates]": { + "iqr": 0.018300908000128402, + "iterations": 1, + "max": 0.6447552070003439, + "mean": 0.5987897021001117, + "median": 0.596316669999851, + "min": 0.5658971729999394, + "rounds": 10, + "stddev": 0.020266233206775505 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[unified_jobs]": { + "iqr": 0.017110222998780955, + "iterations": 1, + "max": 0.6404829670000254, + "mean": 0.6074243255999136, + "median": 0.6040582579998954, + "min": 0.5824695769997561, + "rounds": 10, + "stddev": 0.019860804656387804 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[users]": { + "iqr": 0.012589865999871108, + "iterations": 1, + "max": 0.30070708800030843, + "mean": 0.2801567766999142, + "median": 0.27962559700017664, + "min": 0.2699828269996942, + "rounds": 10, + "stddev": 0.009850085323966229 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_nodes]": { + "iqr": 0.0038878800005477387, + "iterations": 1, + "max": 0.08795158799966885, + "mean": 0.07403220392844949, + "median": 0.07276557799968941, + "min": 0.06972812999993039, + "rounds": 14, + "stddev": 0.004805645360449875 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_template_nodes]": { + "iqr": 0.0053214569998090155, + "iterations": 1, + "max": 0.1973419690002629, + "mean": 0.0827645748572, + "median": 0.07252062150018901, + "min": 0.07018768700072542, + "rounds": 14, + "stddev": 0.03318749492730351 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_job_templates]": { + "iqr": 0.0026864617502724286, + "iterations": 1, + "max": 0.08489979800015135, + "mean": 0.07753878323092067, + "median": 0.07679366800039134, + "min": 0.07368176500040136, + "rounds": 13, + "stddev": 0.0032710486591505405 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_endpoint_response[workflow_jobs]": { + "iqr": 0.0015324419991884497, + "iterations": 1, + "max": 0.07553425500009325, + "mean": 0.07248855085702287, + "median": 0.07201780499963206, + "min": 0.07039047699981893, + "rounds": 14, + "stddev": 0.0015712771972324231 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_creation": { + "iqr": 0.021125644999301585, + "iterations": 1, + "max": 0.323358704000384, + "mean": 0.3043606732999251, + "median": 0.3034965789997841, + "min": 0.28892963900034374, + "rounds": 10, + "stddev": 0.011943715094823787 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_group_creation": { + "iqr": 0.004571244999169721, + "iterations": 1, + "max": 0.24221611800021492, + "mean": 0.22489656210018438, + "median": 0.22346884200032946, + "min": 0.2180190710005263, + "rounds": 10, + "stddev": 0.006653982708616119 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_inventory_source_update": { + "iqr": 0.014473835000899271, + "iterations": 1, + "max": 0.9577174820005894, + "mean": 0.8980187795999882, + "median": 0.929607965500054, + "min": 0.5659871269999712, + "rounds": 10, + "stddev": 0.11711287921937714 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_slicing_workaround": { + "iqr": 0.12561452799945982, + "iterations": 1, + "max": 8.98362829999951, + "mean": 8.780200364899974, + "median": 8.78412779600012, + "min": 8.459455754999908, + "rounds": 10, + "stddev": 0.1409537143944923 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_creation": { + "iqr": 0.024824041000101715, + "iterations": 1, + "max": 1.1475435659995128, + "mean": 1.054246841099939, + "median": 1.043016840999826, + "min": 1.0118832180005484, + "rounds": 10, + "stddev": 0.04190708245878323 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_slicing": { + "iqr": 0.33080408000114403, + "iterations": 1, + "max": 9.492328033000376, + "mean": 9.183380859299996, + "median": 9.08570949749992, + "min": 8.984000387000378, + "rounds": 10, + "stddev": 0.19489658808225244 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_job_template_start_time": { + "iqr": 0.5727856279991101, + "iterations": 1, + "max": 1.7148072149993823, + "mean": 1.3301523682001062, + "median": 1.3027406630003497, + "min": 1.034838008999941, + "rounds": 10, + "stddev": 0.29214603246322207 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_machine_credential_creation": { + "iqr": 0.010440658000334224, + "iterations": 1, + "max": 0.41400267300014093, + "mean": 0.4017492543000117, + "median": 0.40455711300000985, + "min": 0.38070554099977016, + "rounds": 10, + "stddev": 0.010739263267704125 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_nested_empty_workflows_in_workflow": { + "iqr": 0.07552487600150926, + "iterations": 1, + "max": 12.011054894999688, + "mean": 11.546496755200314, + "median": 11.468550067500473, + "min": 11.403476662000685, + "rounds": 10, + "stddev": 0.19976123645470475 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_org_creation": { + "iqr": 0.007258775000082096, + "iterations": 1, + "max": 0.42456348499945307, + "mean": 0.41190965379983024, + "median": 0.410843263499828, + "min": 0.4057220659997256, + "rounds": 10, + "stddev": 0.005978441777347863 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_project_creation": { + "iqr": 0.11285750499973801, + "iterations": 1, + "max": 0.838216674999785, + "mean": 0.5688441418000366, + "median": 0.5329968740002187, + "min": 0.42780234800011385, + "rounds": 10, + "stddev": 0.13234432999825954 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_host_large_inventory": { + "iqr": 0.31282600600025035, + "iterations": 1, + "max": 11.980996977001269, + "mean": 11.63180001170058, + "median": 11.70190846100013, + "min": 11.279163364999476, + "rounds": 10, + "stddev": 0.21705983060685108 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_single_job": { + "iqr": 0.22208135400069295, + "iterations": 1, + "max": 8.133883817999958, + "mean": 7.859452761400189, + "median": 7.829370411000582, + "min": 7.658075274999646, + "rounds": 10, + "stddev": 0.15857939185330955 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-100]": { + "iqr": 0.0, + "iterations": 1, + "max": 261.2323028689989, + "mean": 261.2323028689989, + "median": 261.2323028689989, + "min": 261.2323028689989, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_task_manager_launch_sliced_jt_jobs[tower-300]": { + "iqr": 0.0, + "iterations": 1, + "max": 385.0585781919999, + "mean": 385.0585781919999, + "median": 385.0585781919999, + "min": 385.0585781919999, + "rounds": 1, + "stddev": 0 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_team_creation": { + "iqr": 0.004363357000329415, + "iterations": 1, + "max": 0.2737064629991437, + "mean": 0.266880075699828, + "median": 0.2673320854996746, + "min": 0.2558003849999295, + "rounds": 10, + "stddev": 0.004733004905241722 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_user_creation": { + "iqr": 0.008850257999256428, + "iterations": 1, + "max": 0.6751608170006875, + "mean": 0.6558408117001818, + "median": 0.6558430095001313, + "min": 0.6411633299994719, + "rounds": 10, + "stddev": 0.009118106824348448 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow": { + "iqr": 0.2661390670000401, + "iterations": 1, + "max": 9.670757316000163, + "mean": 9.470276577099867, + "median": 9.448280169999634, + "min": 9.280793309999353, + "rounds": 10, + "stddev": 0.15202563159169838 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_in_workflow_node_start_time": { + "iqr": 0.5163036210005885, + "iterations": 1, + "max": 5.236855375000232, + "mean": 3.4189237352999045, + "median": 3.1697728740000457, + "min": 2.1991375950001384, + "rounds": 10, + "stddev": 1.020031274019219 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_benchmark_workflow_node_start_time": { + "iqr": 0.0894864999991114, + "iterations": 1, + "max": 3.90060523000011, + "mean": 2.0887428583001566, + "median": 1.9525875620001898, + "min": 1.5685169410007802, + "rounds": 10, + "stddev": 0.6598973897290226 + }, + "tests/api/test_api_benchmark_py::TestApiPerformance::test_workflow_multiple_project_use": { + "iqr": 0.4936595670005772, + "iterations": 1, + "max": 27.027984514999844, + "mean": 25.146485302999828, + "median": 25.197155586999543, + "min": 24.08453130899943, + "rounds": 10, + "stddev": 0.829101453183599 + } + }, + "started": "2022-01-17T21:59:19.143649+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-chatty.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-chatty.json new file mode 100644 index 00000000..0e7a7718 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-chatty.json @@ -0,0 +1,636 @@ +{ + "ended": "2022-01-17T21:12:41.027251+00:00", + "id": "run-2022-01-17T20_14_07_784_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 8961.233333333337, + "max": 47764.333333333336, + "mean": 31967.476767676766, + "median": 38099.73333333334, + "min": 1592.9333333333334, + "non_zero_mean": 31967.476767676766, + "non_zero_median": 38099.73333333334, + "percentile25": 30651.433333333334, + "percentile75": 39612.66666666667, + "percentile90": 41131.76666666666, + "percentile99": 44224.12999999997, + "percentile999": 47410.31300000002, + "range": 46171.4, + "samples": 66, + "stdev": 12844.378358933996, + "sum": 2109853.466666667 + } + }, + "cpu": { + "system": { + "iqr": 0.2249999999999993, + "max": 1.0720000000000045, + "mean": 0.8536868686868687, + "median": 1.014333333333334, + "min": 0.04200000000000017, + "non_zero_mean": 0.8536868686868687, + "non_zero_median": 1.014333333333334, + "percentile25": 0.8094999999999998, + "percentile75": 1.034499999999999, + "percentile90": 1.0459999999999994, + "percentile99": 1.0676666666666685, + "percentile999": 1.071566666666671, + "range": 1.0300000000000042, + "samples": 66, + "stdev": 0.30809209899953854, + "sum": 56.34333333333334 + }, + "user": { + "iqr": 2.2085000000000017, + "max": 6.666666666666667, + "mean": 5.108212121212121, + "median": 6.31266666666667, + "min": 0.21799999999999878, + "non_zero_mean": 5.108212121212121, + "non_zero_median": 6.31266666666667, + "percentile25": 4.179333333333332, + "percentile75": 6.387833333333334, + "percentile90": 6.435999999999986, + "percentile99": 6.5869333333333335, + "percentile999": 6.658693333333334, + "range": 6.448666666666668, + "samples": 66, + "stdev": 2.0180915353151465, + "sum": 337.142 + } + }, + "entropy_available_bits": { + "iqr": 7.733333333333333, + "max": 425.4666666666667, + "mean": 27.493939393939396, + "median": 3.9333333333333336, + "min": 0.0, + "non_zero_mean": 41.24090909090909, + "non_zero_median": 7.633333333333333, + "percentile25": 0.0, + "percentile75": 7.733333333333333, + "percentile90": 14.3, + "percentile99": 425.03333333333336, + "percentile999": 425.42333333333335, + "range": 425.4666666666667, + "samples": 66, + "stdev": 92.1785142677972, + "sum": 1814.6000000000004 + }, + "memory": { + "used": { + "iqr": 1420854272.0, + "max": 7612284928.0, + "mean": 6374422590.060606, + "median": 6813771776.0, + "min": 3509809152.0, + "non_zero_mean": 6374422590.060606, + "non_zero_median": 6813771776.0, + "percentile25": 5635820544.0, + "percentile75": 7056674816.0, + "percentile90": 7205480448.0, + "percentile99": 7588477747.2, + "percentile999": 7609904209.92, + "range": 4102475776.0, + "samples": 66, + "stdev": 930739151.0996196, + "sum": 420711890944.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 2.0, + "mean": 0.06666666666666667, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.1, + "non_zero_median": 1.1333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 2.0, + "percentile999": 2.0, + "range": 2.0, + "samples": 66, + "stdev": 0.3463114550973212, + "sum": 4.4 + } + }, + "writes_completed": { + "total": { + "iqr": 2219.316666666666, + "max": 4954.133333333333, + "mean": 3476.170707070707, + "median": 4462.4, + "min": 6.666666666666666, + "non_zero_mean": 3476.170707070707, + "non_zero_median": 4462.4, + "percentile25": 2403.7000000000003, + "percentile75": 4623.016666666666, + "percentile90": 4688.4, + "percentile99": 4907.94, + "percentile999": 4949.514, + "range": 4947.466666666666, + "samples": 66, + "stdev": 1646.1377372090606, + "sum": 229427.26666666672 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 287.5333333333333, + "max": 2310.9333333333334, + "mean": 1657.3959595959595, + "median": 1739.4333333333334, + "min": 694.7333333333333, + "non_zero_mean": 1657.3959595959595, + "non_zero_median": 1739.4333333333334, + "percentile25": 1548.0833333333335, + "percentile75": 1835.6166666666668, + "percentile90": 1963.9666666666667, + "percentile99": 2284.9333333333334, + "percentile999": 2308.3333333333335, + "range": 1616.2, + "samples": 66, + "stdev": 329.648382242425, + "sum": 109388.1333333333 + } + }, + "cpu": { + "system": { + "iqr": 0.029000000000000276, + "max": 0.24400000000000027, + "mean": 0.14561616161616162, + "median": 0.1536666666666665, + "min": 0.02933333333333342, + "non_zero_mean": 0.14561616161616162, + "non_zero_median": 0.1536666666666665, + "percentile25": 0.13699999999999982, + "percentile75": 0.1660000000000001, + "percentile90": 0.18999999999999997, + "percentile99": 0.24010000000000006, + "percentile999": 0.24361000000000027, + "range": 0.21466666666666684, + "samples": 66, + "stdev": 0.04623730104891513, + "sum": 9.610666666666667 + }, + "user": { + "iqr": 0.17083333333333298, + "max": 0.4226666666666664, + "mean": 0.23315151515151514, + "median": 0.18933333333333263, + "min": 0.033999999999999864, + "non_zero_mean": 0.23315151515151514, + "non_zero_median": 0.18933333333333263, + "percentile25": 0.17116666666666744, + "percentile75": 0.3420000000000004, + "percentile90": 0.3550000000000001, + "percentile99": 0.41183333333333333, + "percentile999": 0.42158333333333314, + "range": 0.38866666666666655, + "samples": 66, + "stdev": 0.103186068459675, + "sum": 15.388 + } + }, + "entropy_available_bits": { + "iqr": 1.7500000000000002, + "max": 195.06666666666666, + "mean": 12.032323232323233, + "median": 2.1333333333333333, + "min": 0.6, + "non_zero_mean": 12.032323232323233, + "non_zero_median": 2.1333333333333333, + "percentile25": 1.4666666666666666, + "percentile75": 3.216666666666667, + "percentile90": 4.8, + "percentile99": 187.39666666666662, + "percentile999": 194.2996666666667, + "range": 194.46666666666667, + "samples": 66, + "stdev": 39.17682420155512, + "sum": 794.1333333333337 + }, + "memory": { + "used": { + "iqr": 23059456.0, + "max": 528551936.0, + "mean": 451365174.3030303, + "median": 455165952.0, + "min": 361291776.0, + "non_zero_mean": 451365174.3030303, + "non_zero_median": 455165952.0, + "percentile25": 441291776.0, + "percentile75": 464351232.0, + "percentile90": 515489792.0, + "percentile99": 527830425.59999996, + "percentile999": 528479784.96, + "range": 167260160.0, + "samples": 66, + "stdev": 42266857.170084834, + "sum": 29790101504.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 1874056.5333333334, + "max": 3410056.533333333, + "mean": 781831.2404040404, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2064034.4746666667, + "non_zero_median": 2171972.2666666666, + "percentile25": 0.0, + "percentile75": 1874056.5333333334, + "percentile90": 3015475.2, + "percentile99": 3374912.853333333, + "percentile999": 3406542.1653333334, + "range": 3410056.533333333, + "samples": 66, + "stdev": 1207302.5491269634, + "sum": 51600861.866666675 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 0.225525, + "mean": 0.010294515151515152, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.04529586666666667, + "non_zero_median": 0.012288, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.015941, + "percentile99": 0.2022861999999998, + "percentile999": 0.22320112000000009, + "range": 0.225525, + "samples": 66, + "stdev": 0.037914945894621487, + "sum": 0.679438 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.00584, + "max": 0.768448, + "mean": 0.10943781818181818, + "median": 0.0, + "min": -0.000516, + "non_zero_mean": 0.28891584, + "non_zero_median": 0.22244, + "percentile25": 0.0, + "percentile75": 0.00584, + "percentile90": 0.607024, + "percentile99": 0.7337015999999996, + "percentile999": 0.7649733600000002, + "range": 0.768964, + "samples": 66, + "stdev": 0.23481367891834246, + "sum": 7.2228959999999995 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 13.933333333333334, + "mean": 0.4222222222222222, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9.288888888888888, + "non_zero_median": 9.733333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 11.203333333333308, + "percentile999": 13.660333333333343, + "range": 13.933333333333334, + "samples": 66, + "stdev": 2.129462299879875, + "sum": 27.866666666666667 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9993.666666666666, + "mean": 453.46262626262626, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 9976.177777777777, + "non_zero_median": 9977.2, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9982.963333333333, + "percentile999": 9992.596333333333, + "range": 9993.666666666666, + "samples": 66, + "stdev": 2093.95300587693, + "sum": 29928.533333333333 + }, + "pg_stat_database_blks_hit": { + "iqr": 3075.2000000000044, + "max": 37377.13333333333, + "mean": 21669.409090909092, + "median": 23052.533333333333, + "min": 7146.333333333333, + "non_zero_mean": 21669.409090909092, + "non_zero_median": 23052.533333333333, + "percentile25": 21019.666666666664, + "percentile75": 24094.86666666667, + "percentile90": 25532.566666666666, + "percentile99": 33971.04666666663, + "percentile999": 37036.52466666668, + "range": 30230.8, + "samples": 66, + "stdev": 5801.366416761048, + "sum": 1430180.9999999998 + }, + "pg_stat_database_blks_read": { + "iqr": 74.48333333333332, + "max": 149.4, + "mean": 97.4040404040404, + "median": 126.93333333333334, + "min": 0.0, + "non_zero_mean": 100.44791666666667, + "non_zero_median": 126.96666666666667, + "percentile25": 59.96666666666667, + "percentile75": 134.45, + "percentile90": 140.36666666666667, + "percentile99": 146.3233333333333, + "percentile999": 149.09233333333336, + "range": 149.4, + "samples": 66, + "stdev": 49.54282737296837, + "sum": 6428.666666666668 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 74.48333333333332, + "max": 149.4, + "mean": 97.4040404040404, + "median": 126.93333333333334, + "min": 0.0, + "non_zero_mean": 100.44791666666667, + "non_zero_median": 126.96666666666667, + "percentile25": 59.96666666666667, + "percentile75": 134.45, + "percentile90": 140.36666666666667, + "percentile99": 146.3233333333333, + "percentile999": 149.09233333333336, + "range": 149.4, + "samples": 66, + "stdev": 49.54282737296837, + "sum": 6428.666666666668 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.1, + "max": 1.7333333333333334, + "mean": 0.2191919191919192, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.8509803921568627, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.1, + "percentile90": 0.9333333333333333, + "percentile99": 1.6899999999999997, + "percentile999": 1.7290000000000003, + "range": 1.7333333333333334, + "samples": 66, + "stdev": 0.43406488963325335, + "sum": 14.466666666666667 + }, + "pg_stat_database_tup_fetched": { + "iqr": 3037.4833333333336, + "max": 23603.2, + "mean": 7264.151515151515, + "median": 5758.333333333334, + "min": 3781.9333333333334, + "non_zero_mean": 7264.151515151515, + "non_zero_median": 5758.333333333334, + "percentile25": 5070.583333333334, + "percentile75": 8108.0666666666675, + "percentile90": 11781.233333333334, + "percentile99": 22299.29999999999, + "percentile999": 23472.810000000005, + "range": 19821.266666666666, + "samples": 66, + "stdev": 3930.062754855224, + "sum": 479433.99999999994 + }, + "pg_stat_database_tup_inserted": { + "iqr": 389.41666666666663, + "max": 802.9333333333333, + "mean": 517.779797979798, + "median": 675.3333333333334, + "min": 0.0, + "non_zero_mean": 542.4359788359789, + "non_zero_median": 678.9333333333333, + "percentile25": 322.1166666666667, + "percentile75": 711.5333333333333, + "percentile90": 743.1666666666666, + "percentile99": 779.7066666666665, + "percentile999": 800.6106666666667, + "range": 802.9333333333333, + "samples": 66, + "stdev": 261.5109550664897, + "sum": 34173.46666666667 + }, + "pg_stat_database_tup_returned": { + "iqr": 3208.5499999999993, + "max": 31161.733333333334, + "mean": 9125.99292929293, + "median": 7088.833333333334, + "min": 4696.066666666667, + "non_zero_mean": 9125.99292929293, + "non_zero_median": 7088.833333333334, + "percentile25": 6297.116666666667, + "percentile75": 9505.666666666666, + "percentile90": 16879.933333333334, + "percentile99": 28044.63666666664, + "percentile999": 30850.02366666668, + "range": 26465.666666666668, + "samples": 66, + "stdev": 5193.040327024534, + "sum": 602315.5333333333 + }, + "pg_stat_database_tup_updated": { + "iqr": 3.4499999999999997, + "max": 136.06666666666666, + "mean": 9.91010101010101, + "median": 0.4666666666666667, + "min": 0.0, + "non_zero_mean": 11.892121212121213, + "non_zero_median": 0.8666666666666667, + "percentile25": 0.08333333333333333, + "percentile75": 3.533333333333333, + "percentile90": 29.96666666666667, + "percentile99": 109.8933333333331, + "percentile999": 133.4493333333334, + "range": 136.06666666666666, + "samples": 66, + "stdev": 24.5497227652959, + "sum": 654.0666666666666 + }, + "pg_stat_database_xact_commit": { + "iqr": 10.783333333333331, + "max": 152.86666666666667, + "mean": 58.647474747474746, + "median": 50.766666666666666, + "min": 7.666666666666667, + "non_zero_mean": 58.647474747474746, + "non_zero_median": 50.766666666666666, + "percentile25": 47.6, + "percentile75": 58.38333333333333, + "percentile90": 101.30000000000001, + "percentile99": 141.25333333333325, + "percentile999": 151.70533333333339, + "range": 145.20000000000002, + "samples": 66, + "stdev": 27.999171575416458, + "sum": 3870.733333333334 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 66, + "stdev": 0.0, + "sum": 4.400000000000004 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 1.2, + "mean": 0.022222222222222223, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.7333333333333333, + "non_zero_median": 0.7333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.593333333333328, + "percentile999": 1.1393333333333353, + "range": 1.2, + "samples": 66, + "stdev": 0.15081922538734494, + "sum": 1.4666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 26.483333333333334, + "max": 169.53333333333333, + "mean": 78.53636363636363, + "median": 76.03333333333333, + "min": 4.866666666666667, + "non_zero_mean": 78.53636363636363, + "non_zero_median": 76.03333333333333, + "percentile25": 67.75, + "percentile75": 94.23333333333333, + "percentile90": 109.53333333333333, + "percentile99": 147.6499999999998, + "percentile999": 167.34500000000008, + "range": 164.66666666666666, + "samples": 66, + "stdev": 29.904484829172578, + "sum": 5183.399999999999 + } + } + } + }, + "name": "chatty_tasks.yml performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 2, + "job_ids_str": "3,4,5,6,7,9,8,11,12,10,18,20,21,19,22,23,24,25,26,27,28,29,31,33,32,34,35,30,36,37,38,40,41,43,44,42,39,45,46,47,48,49,50,53,52,51,54,55,56,57", + "job_template_id": 9, + "project_id": 8, + "started": "2022-01-17T20:53:37.475313Z", + "test_chatty_concurency": 10, + "test_chatty_hosts": 100, + "test_chatty_message_size": 1024, + "test_chatty_num_messages": 100, + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_repeat": 5, + "test_chatty_wait": 1800 + }, + "result": "PASS", + "results": { + "created": "2022-01-17T20:53:38.388209Z 2022-01-17T20:53:38.436284Z 2022-01-17T20:53:38.121586Z 2022-01-17T20:53:37.628635Z 2022-01-17T20:53:37.615807Z 2022-01-17T20:53:37.605481Z 2022-01-17T20:53:37.475313Z 2022-01-17T20:53:38.785188Z 2022-01-17T20:53:38.718089Z 2022-01-17T20:53:38.608078Z 2022-01-17T20:57:52.766393Z 2022-01-17T20:57:52.445943Z 2022-01-17T20:57:51.984592Z 2022-01-17T20:57:51.975100Z 2022-01-17T20:57:51.914405Z 2022-01-17T20:57:51.896626Z 2022-01-17T20:57:51.891346Z 2022-01-17T20:57:51.875783Z 2022-01-17T20:57:51.869701Z 2022-01-17T20:57:51.828524Z 2022-01-17T21:00:57.657667Z 2022-01-17T21:00:57.657106Z 2022-01-17T21:00:57.246128Z 2022-01-17T21:00:57.223901Z 2022-01-17T21:00:57.214843Z 2022-01-17T21:00:57.207283Z 2022-01-17T21:00:57.199885Z 2022-01-17T21:00:57.155861Z 2022-01-17T21:00:57.073514Z 2022-01-17T21:00:57.044431Z 2022-01-17T21:04:12.103233Z 2022-01-17T21:04:11.967620Z 2022-01-17T21:04:11.769655Z 2022-01-17T21:04:11.264037Z 2022-01-17T21:04:11.234925Z 2022-01-17T21:04:11.229345Z 2022-01-17T21:04:11.213167Z 2022-01-17T21:04:11.210902Z 2022-01-17T21:04:11.099857Z 2022-01-17T21:04:11.080623Z 2022-01-17T21:07:16.377217Z 2022-01-17T21:07:16.175256Z 2022-01-17T21:07:15.811460Z 2022-01-17T21:07:15.796849Z 2022-01-17T21:07:15.726721Z 2022-01-17T21:07:15.717350Z 2022-01-17T21:07:15.706017Z 2022-01-17T21:07:15.704455Z 2022-01-17T21:07:15.662913Z 2022-01-17T21:07:15.591450Z", + "created_diff": 0.9338713999999999, + "ended": "2022-01-17T21:09:56.373817Z", + "error_job_ids": " []", + "event_first": "2022-01-17T20:55:23.167072Z 2022-01-17T20:53:50.047859Z 2022-01-17T20:53:49.722370Z 2022-01-17T20:54:50.548081Z 2022-01-17T20:53:44.861692Z 2022-01-17T20:54:27.631492Z 2022-01-17T20:53:43.596472Z 2022-01-17T20:55:05.109182Z 2022-01-17T20:53:52.068191Z 2022-01-17T20:54:37.482478Z 2022-01-17T20:58:04.833063Z 2022-01-17T20:58:08.689473Z 2022-01-17T20:58:03.164270Z 2022-01-17T20:58:00.955483Z 2022-01-17T20:57:59.055049Z 2022-01-17T20:58:00.203534Z 2022-01-17T20:58:01.616402Z 2022-01-17T20:57:59.549748Z 2022-01-17T20:57:58.275180Z 2022-01-17T20:57:58.799583Z 2022-01-17T21:01:27.918756Z 2022-01-17T21:01:24.853484Z 2022-01-17T21:01:24.361194Z 2022-01-17T21:01:06.321764Z 2022-01-17T21:01:03.867810Z 2022-01-17T21:01:04.085454Z 2022-01-17T21:01:03.106060Z 2022-01-17T21:01:04.287574Z 2022-01-17T21:01:02.851194Z 2022-01-17T21:01:03.110091Z 2022-01-17T21:04:24.602758Z 2022-01-17T21:04:25.006538Z 2022-01-17T21:04:23.725397Z 2022-01-17T21:04:22.208990Z 2022-01-17T21:04:18.087077Z 2022-01-17T21:04:21.446599Z 2022-01-17T21:04:19.874084Z 2022-01-17T21:04:20.301527Z 2022-01-17T21:04:17.655636Z 2022-01-17T21:04:17.980814Z 2022-01-17T21:07:31.971623Z 2022-01-17T21:07:32.937726Z 2022-01-17T21:07:30.465214Z 2022-01-17T21:07:24.718542Z 2022-01-17T21:07:22.216459Z 2022-01-17T21:07:24.538414Z 2022-01-17T21:07:22.216459Z 2022-01-17T21:07:22.367635Z 2022-01-17T21:07:21.968264Z 2022-01-17T21:07:23.142981Z", + "event_last": "2022-01-17T20:57:32.097455Z 2022-01-17T20:56:24.196092Z 2022-01-17T20:56:23.368756Z 2022-01-17T20:57:11.364234Z 2022-01-17T20:56:17.809127Z 2022-01-17T20:56:56.202040Z 2022-01-17T20:56:16.931628Z 2022-01-17T20:57:20.425332Z 2022-01-17T20:56:26.175398Z 2022-01-17T20:57:02.357833Z 2022-01-17T21:00:29.720930Z 2022-01-17T21:00:31.149588Z 2022-01-17T21:00:27.490481Z 2022-01-17T21:00:27.162282Z 2022-01-17T21:00:25.372008Z 2022-01-17T21:00:25.842735Z 2022-01-17T21:00:26.863406Z 2022-01-17T21:00:26.687380Z 2022-01-17T21:00:24.378152Z 2022-01-17T21:00:25.159487Z 2022-01-17T21:03:50.839223Z 2022-01-17T21:03:47.351643Z 2022-01-17T21:03:49.180895Z 2022-01-17T21:03:33.953843Z 2022-01-17T21:03:35.875722Z 2022-01-17T21:03:32.090391Z 2022-01-17T21:03:35.789156Z 2022-01-17T21:03:29.898281Z 2022-01-17T21:03:33.933426Z 2022-01-17T21:03:28.975630Z 2022-01-17T21:06:47.847506Z 2022-01-17T21:06:48.433185Z 2022-01-17T21:06:47.418112Z 2022-01-17T21:06:47.528342Z 2022-01-17T21:06:42.569937Z 2022-01-17T21:06:46.444135Z 2022-01-17T21:06:44.985221Z 2022-01-17T21:06:45.018382Z 2022-01-17T21:06:41.819934Z 2022-01-17T21:06:44.563644Z 2022-01-17T21:09:53.214187Z 2022-01-17T21:09:56.373817Z 2022-01-17T21:09:52.730631Z 2022-01-17T21:09:49.853263Z 2022-01-17T21:09:48.153618Z 2022-01-17T21:09:49.180668Z 2022-01-17T21:09:48.204919Z 2022-01-17T21:09:48.885488Z 2022-01-17T21:09:47.247099Z 2022-01-17T21:09:47.442035Z", + "failed_job_ids": " []", + "finished": "2022-01-17T20:57:36.422272Z 2022-01-17T20:56:27.601748Z 2022-01-17T20:56:27.229408Z 2022-01-17T20:57:15.039070Z 2022-01-17T20:56:21.461922Z 2022-01-17T20:56:59.679793Z 2022-01-17T20:56:21.276395Z 2022-01-17T20:57:24.367609Z 2022-01-17T20:56:29.999546Z 2022-01-17T20:57:06.256472Z 2022-01-17T21:00:33.494772Z 2022-01-17T21:00:34.594034Z 2022-01-17T21:00:31.545077Z 2022-01-17T21:00:30.865009Z 2022-01-17T21:00:29.022795Z 2022-01-17T21:00:30.351789Z 2022-01-17T21:00:31.194248Z 2022-01-17T21:00:31.052629Z 2022-01-17T21:00:28.887269Z 2022-01-17T21:00:29.594515Z 2022-01-17T21:03:54.224259Z 2022-01-17T21:03:51.033164Z 2022-01-17T21:03:52.864345Z 2022-01-17T21:03:37.698062Z 2022-01-17T21:03:39.484339Z 2022-01-17T21:03:36.527904Z 2022-01-17T21:03:39.956561Z 2022-01-17T21:03:33.435141Z 2022-01-17T21:03:37.926577Z 2022-01-17T21:03:32.677159Z 2022-01-17T21:06:52.024747Z 2022-01-17T21:06:51.999362Z 2022-01-17T21:06:51.788551Z 2022-01-17T21:06:51.359447Z 2022-01-17T21:06:46.398935Z 2022-01-17T21:06:49.980406Z 2022-01-17T21:06:48.347465Z 2022-01-17T21:06:48.557540Z 2022-01-17T21:06:46.026305Z 2022-01-17T21:06:48.173252Z 2022-01-17T21:09:57.307722Z 2022-01-17T21:09:58.337869Z 2022-01-17T21:09:56.848269Z 2022-01-17T21:09:53.275565Z 2022-01-17T21:09:51.970494Z 2022-01-17T21:09:52.840953Z 2022-01-17T21:09:51.529933Z 2022-01-17T21:09:52.709392Z 2022-01-17T21:09:51.419162Z 2022-01-17T21:09:51.606881Z", + "finished_diff": 23.0633782, + "host_status_counts": "{'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100} {'ok': 100}", + "jobs_duration": { + "max": 236.186578, + "mean": 163.8144701, + "min": 153.422123 + }, + "jobs_events_duration": { + "max": 154.148233, + "mean": 145.69967770000002, + "min": 128.930383 + }, + "jobs_events_lag": { + "max": -1.964052, + "mean": -3.834189120000001, + "min": -4.509117 + }, + "jobs_waiting": { + "max": 20.305432, + "mean": 2.6117574800000005, + "min": 0.903758 + }, + "modified": "2022-01-17T20:53:39.752287Z 2022-01-17T20:53:39.787163Z 2022-01-17T20:53:39.717158Z 2022-01-17T20:53:38.190155Z 2022-01-17T20:53:38.152975Z 2022-01-17T20:53:38.110874Z 2022-01-17T20:53:38.065230Z 2022-01-17T20:53:39.897353Z 2022-01-17T20:53:39.857665Z 2022-01-17T20:53:39.823032Z 2022-01-17T20:57:53.933551Z 2022-01-17T20:57:53.850661Z 2022-01-17T20:57:52.678480Z 2022-01-17T20:57:52.648885Z 2022-01-17T20:57:52.619732Z 2022-01-17T20:57:52.590131Z 2022-01-17T20:57:52.557754Z 2022-01-17T20:57:52.521761Z 2022-01-17T20:57:52.483393Z 2022-01-17T20:57:52.430945Z 2022-01-17T21:01:16.615509Z 2022-01-17T21:01:16.392738Z 2022-01-17T21:01:16.250274Z 2022-01-17T21:00:57.817447Z 2022-01-17T21:00:57.782452Z 2022-01-17T21:00:57.743086Z 2022-01-17T21:00:57.711353Z 2022-01-17T21:00:57.679547Z 2022-01-17T21:00:57.645984Z 2022-01-17T21:00:57.608426Z 2022-01-17T21:04:13.179487Z 2022-01-17T21:04:13.141173Z 2022-01-17T21:04:13.103502Z 2022-01-17T21:04:11.907996Z 2022-01-17T21:04:11.872739Z 2022-01-17T21:04:11.839510Z 2022-01-17T21:04:11.809224Z 2022-01-17T21:04:11.778654Z 2022-01-17T21:04:11.747558Z 2022-01-17T21:04:11.715602Z 2022-01-17T21:07:19.562268Z 2022-01-17T21:07:19.525064Z 2022-01-17T21:07:19.486368Z 2022-01-17T21:07:16.346351Z 2022-01-17T21:07:16.316542Z 2022-01-17T21:07:16.285354Z 2022-01-17T21:07:16.251551Z 2022-01-17T21:07:16.222413Z 2022-01-17T21:07:16.192419Z 2022-01-17T21:07:16.157949Z", + "modified_diff": 5.4420032, + "started": "2022-01-17T20:53:40.235694Z 2022-01-17T20:53:40.388717Z 2022-01-17T20:53:40.180049Z 2022-01-17T20:53:38.688363Z 2022-01-17T20:53:38.520781Z 2022-01-17T20:53:38.556414Z 2022-01-17T20:53:38.379071Z 2022-01-17T20:53:40.573996Z 2022-01-17T20:53:40.634662Z 2022-01-17T20:53:40.420749Z 2022-01-17T20:57:54.596623Z 2022-01-17T20:57:54.420518Z 2022-01-17T20:57:53.450202Z 2022-01-17T20:57:53.349496Z 2022-01-17T20:57:53.237176Z 2022-01-17T20:57:53.171498Z 2022-01-17T20:57:53.086288Z 2022-01-17T20:57:53.000574Z 2022-01-17T20:57:52.933660Z 2022-01-17T20:57:52.842667Z 2022-01-17T21:01:17.963099Z 2022-01-17T21:01:17.611041Z 2022-01-17T21:01:17.205139Z 2022-01-17T21:00:58.544667Z 2022-01-17T21:00:58.450866Z 2022-01-17T21:00:58.347317Z 2022-01-17T21:00:58.262206Z 2022-01-17T21:00:58.198488Z 2022-01-17T21:00:58.107644Z 2022-01-17T21:00:58.021605Z 2022-01-17T21:04:13.709130Z 2022-01-17T21:04:13.594318Z 2022-01-17T21:04:13.459233Z 2022-01-17T21:04:12.890616Z 2022-01-17T21:04:12.661277Z 2022-01-17T21:04:12.609831Z 2022-01-17T21:04:12.466860Z 2022-01-17T21:04:12.385647Z 2022-01-17T21:04:12.266262Z 2022-01-17T21:04:12.177365Z 2022-01-17T21:07:20.183232Z 2022-01-17T21:07:19.982799Z 2022-01-17T21:07:19.835906Z 2022-01-17T21:07:17.091459Z 2022-01-17T21:07:17.001427Z 2022-01-17T21:07:16.917514Z 2022-01-17T21:07:16.852792Z 2022-01-17T21:07:16.770717Z 2022-01-17T21:07:16.695416Z 2022-01-17T21:07:16.611557Z", + "started_diff": 5.810896199999999, + "successful_job_ids": " [9, 8, 7, 6, 5, 4, 3, 12, 11, 10, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48]" + }, + "started": "2022-01-17T21:12:38.507998+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-launching.json b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-launching.json new file mode 100644 index 00000000..c3803ba2 --- /dev/null +++ b/DELME/testing_sd_dir/status-data-run-2022-01-17T20_14_07_784_0000-launching.json @@ -0,0 +1,634 @@ +{ + "ended": "2022-01-17T21:58:16.933599+00:00", + "id": "run-2022-01-17T20_14_07_784_0000", + "measurements": { + "cluster": { + "context_switches": { + "total": { + "iqr": 17384.6, + "max": 27758.4, + "mean": 9928.490043290043, + "median": 4740.599999999999, + "min": 1451.4, + "non_zero_mean": 9928.490043290043, + "non_zero_median": 4740.599999999999, + "percentile25": 1776.0, + "percentile75": 19160.6, + "percentile90": 23609.253333333338, + "percentile99": 27741.528, + "percentile999": 27756.7128, + "range": 26307.0, + "samples": 77, + "stdev": 9242.253513278856, + "sum": 764493.7333333334 + } + }, + "cpu": { + "system": { + "iqr": 1.3886666666666656, + "max": 1.9579999999999984, + "mean": 0.689038961038961, + "median": 0.2646666666666666, + "min": 0.027999999999997274, + "non_zero_mean": 0.689038961038961, + "non_zero_median": 0.2646666666666666, + "percentile25": 0.03733333333333538, + "percentile75": 1.426000000000001, + "percentile90": 1.6870666666666683, + "percentile99": 1.8652799999999985, + "percentile999": 1.9487279999999993, + "range": 1.930000000000001, + "samples": 77, + "stdev": 0.7073088648768722, + "sum": 53.05599999999999 + }, + "user": { + "iqr": 5.765333333333365, + "max": 6.529333333333322, + "mean": 2.870995670995671, + "median": 1.3673333333333404, + "min": 0.09066666666666001, + "non_zero_mean": 2.870995670995671, + "non_zero_median": 1.3673333333333404, + "percentile25": 0.31666666666663634, + "percentile75": 6.082000000000002, + "percentile90": 6.27480000000001, + "percentile99": 6.506026666666665, + "percentile999": 6.5270026666666565, + "range": 6.438666666666662, + "samples": 77, + "stdev": 2.7400063615057757, + "sum": 221.06666666666666 + } + }, + "entropy_available_bits": { + "iqr": 3.866666666666667, + "max": 422.8666666666667, + "mean": 23.681385281385282, + "median": 0.7333333333333334, + "min": 0.0, + "non_zero_mean": 26.427053140096618, + "non_zero_median": 0.7333333333333334, + "percentile25": 0.5333333333333333, + "percentile75": 4.4, + "percentile90": 5.626666666666667, + "percentile99": 422.6133333333333, + "percentile999": 422.84133333333335, + "range": 422.8666666666667, + "samples": 77, + "stdev": 93.63331654729734, + "sum": 1823.466666666667 + }, + "memory": { + "used": { + "iqr": 5947432960.0, + "max": 17843613696.0, + "mean": 9054804952.103895, + "median": 6847750144.0, + "min": 5065928704.0, + "non_zero_mean": 9054804952.103895, + "non_zero_median": 6847750144.0, + "percentile25": 6274232320.0, + "percentile75": 12221665280.0, + "percentile90": 15457424179.200005, + "percentile99": 17646136852.48, + "percentile999": 17823866011.648003, + "range": 12777684992.0, + "samples": 77, + "stdev": 3916981101.375282, + "sum": 697219981312.0 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003463203463203463, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06399999999999864, + "percentile999": 0.24640000000000176, + "range": 0.26666666666666666, + "samples": 77, + "stdev": 0.030389487055903455, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 843.6666666666667, + "max": 1628.1333333333332, + "mean": 456.0900432900433, + "median": 95.26666666666667, + "min": 0.0, + "non_zero_mean": 462.0912280701754, + "non_zero_median": 97.9, + "percentile25": 3.266666666666667, + "percentile75": 846.9333333333334, + "percentile90": 1444.1600000000003, + "percentile99": 1607.9679999999998, + "percentile999": 1626.1168, + "range": 1628.1333333333332, + "samples": 77, + "stdev": 573.0785603623015, + "sum": 35118.93333333334 + } + } + }, + "database": { + "context_switches": { + "total": { + "iqr": 2069.4666666666667, + "max": 3952.5333333333333, + "mean": 1728.569696969697, + "median": 1042.0, + "min": 460.8, + "non_zero_mean": 1728.569696969697, + "non_zero_median": 1042.0, + "percentile25": 677.6666666666666, + "percentile75": 2747.133333333333, + "percentile90": 3181.56, + "percentile99": 3762.7866666666655, + "percentile999": 3933.558666666668, + "range": 3491.733333333333, + "samples": 77, + "stdev": 1127.3424453419839, + "sum": 133099.86666666667 + } + }, + "cpu": { + "system": { + "iqr": 0.21400000000000102, + "max": 0.42399999999999993, + "mean": 0.13598268398268398, + "median": 0.051333333333334015, + "min": 0.012666666666666515, + "non_zero_mean": 0.13598268398268398, + "non_zero_median": 0.051333333333334015, + "percentile25": 0.02599999999999909, + "percentile75": 0.2400000000000001, + "percentile90": 0.300266666666667, + "percentile99": 0.4128533333333333, + "percentile999": 0.4228853333333334, + "range": 0.41133333333333344, + "samples": 77, + "stdev": 0.12415779311403317, + "sum": 10.470666666666663 + }, + "user": { + "iqr": 0.2826666666666692, + "max": 0.8493333333333349, + "mean": 0.2061125541125541, + "median": 0.09599999999999984, + "min": 0.01000000000000038, + "non_zero_mean": 0.2061125541125541, + "non_zero_median": 0.09599999999999984, + "percentile25": 0.040666666666664734, + "percentile75": 0.3233333333333339, + "percentile90": 0.4775999999999987, + "percentile99": 0.8305866666666664, + "percentile999": 0.8474586666666682, + "range": 0.8393333333333346, + "samples": 77, + "stdev": 0.2041605950431932, + "sum": 15.870666666666663 + } + }, + "entropy_available_bits": { + "iqr": 3.1333333333333333, + "max": 212.8, + "mean": 11.883116883116884, + "median": 0.4, + "min": 0.0, + "non_zero_mean": 17.264150943396228, + "non_zero_median": 0.6666666666666666, + "percentile25": 0.0, + "percentile75": 3.1333333333333333, + "percentile90": 4.253333333333334, + "percentile99": 212.344, + "percentile999": 212.7544, + "range": 212.8, + "samples": 77, + "stdev": 46.34586749033904, + "sum": 915.0000000000001 + }, + "memory": { + "used": { + "iqr": 443179008.0, + "max": 1132986368.0, + "mean": 580309683.5324675, + "median": 403533824.0, + "min": 362504192.0, + "non_zero_mean": 580309683.5324675, + "non_zero_median": 403533824.0, + "percentile25": 377180160.0, + "percentile75": 820359168.0, + "percentile90": 1003118592.0000001, + "percentile99": 1086755799.0399997, + "percentile999": 1128363311.1040003, + "range": 770482176.0, + "samples": 77, + "stdev": 258148473.67255083, + "sum": 44683845632.0 + } + }, + "postgresql": { + "pg_database_size_bytes": { + "iqr": 0.0, + "max": 193331.2, + "mean": 16128.664935064935, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 65363.53684210526, + "non_zero_median": 44236.8, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 52101.12000000005, + "percentile99": 186690.21866666662, + "percentile999": 192667.10186666672, + "range": 193331.2, + "samples": 77, + "stdev": 41166.395960246344, + "sum": 1241907.2 + }, + "pg_stat_activity_max_tx_duration-active": { + "iqr": 0.0, + "max": 2.128648, + "mean": 0.04360977922077922, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.3357953, + "non_zero_median": 0.0732325, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.002689200000000001, + "percentile99": 1.0380046799999927, + "percentile999": 2.0195836680000094, + "range": 2.128648, + "samples": 77, + "stdev": 0.2553322482866621, + "sum": 3.3579530000000006 + }, + "pg_stat_activity_max_tx_duration-idle_in_transaction": { + "iqr": 0.686749, + "max": 7.267065, + "mean": 0.584988961038961, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 1.2512263888888888, + "non_zero_median": 0.690619, + "percentile25": 0.0, + "percentile75": 0.686749, + "percentile90": 0.9985666000000029, + "percentile99": 6.201090519999993, + "percentile999": 7.160467552000009, + "range": 7.267065, + "samples": 77, + "stdev": 1.31364627036384, + "sum": 45.04415000000001 + }, + "pg_stat_bgwriter_checkpoint_sync_time": { + "iqr": 0.0, + "max": 5.733333333333333, + "mean": 0.154978354978355, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 2.9833333333333334, + "non_zero_median": 2.3333333333333335, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 3.25066666666665, + "percentile999": 5.485066666666688, + "range": 5.733333333333333, + "samples": 77, + "stdev": 0.763451807127359, + "sum": 11.933333333333334 + }, + "pg_stat_bgwriter_checkpoint_write_time": { + "iqr": 0.0, + "max": 9998.8, + "mean": 435.04415584415585, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 8374.6, + "non_zero_median": 8273.966666666667, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 9623.15733333333, + "percentile999": 9961.235733333337, + "range": 9998.8, + "samples": 77, + "stdev": 1897.6232543658361, + "sum": 33498.399999999994 + }, + "pg_stat_database_blks_hit": { + "iqr": 21972.866666666665, + "max": 104577.26666666666, + "mean": 21831.460606060606, + "median": 11215.6, + "min": 1437.0666666666666, + "non_zero_mean": 21831.460606060606, + "non_zero_median": 11215.6, + "percentile25": 7686.2, + "percentile75": 29659.066666666666, + "percentile90": 45728.22666666668, + "percentile99": 100392.25066666664, + "percentile999": 104158.7650666667, + "range": 103140.2, + "samples": 77, + "stdev": 22399.50276540638, + "sum": 1681022.466666666 + }, + "pg_stat_database_blks_read": { + "iqr": 3.533333333333333, + "max": 12.6, + "mean": 2.006926406926407, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.3594202898550725, + "non_zero_median": 2.9, + "percentile25": 0.0, + "percentile75": 3.533333333333333, + "percentile90": 5.520000000000002, + "percentile99": 11.738666666666662, + "percentile999": 12.513866666666674, + "range": 12.6, + "samples": 77, + "stdev": 2.890488220676487, + "sum": 154.53333333333333 + }, + "pg_stat_database_conflicts": { + "iqr": 0.0, + "max": 0.0, + "mean": 0.0, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.0, + "non_zero_median": 0.0, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.0, + "percentile999": 0.0, + "range": 0.0, + "samples": 77, + "stdev": 0.0, + "sum": 0.0 + }, + "pg_stat_database_deadlocks": { + "iqr": 3.533333333333333, + "max": 12.6, + "mean": 2.006926406926407, + "median": 0.5333333333333333, + "min": 0.0, + "non_zero_mean": 3.3594202898550725, + "non_zero_median": 2.9, + "percentile25": 0.0, + "percentile75": 3.533333333333333, + "percentile90": 5.520000000000002, + "percentile99": 11.738666666666662, + "percentile999": 12.513866666666674, + "range": 12.6, + "samples": 77, + "stdev": 2.890488220676487, + "sum": 154.53333333333333 + }, + "pg_stat_database_tup_deleted": { + "iqr": 0.0, + "max": 2.1333333333333333, + "mean": 0.1619047619047619, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.958974358974359, + "non_zero_median": 0.9333333333333333, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.9333333333333333, + "percentile99": 1.3733333333333282, + "percentile999": 2.05733333333334, + "range": 2.1333333333333333, + "samples": 77, + "stdev": 0.4012303550678875, + "sum": 12.466666666666667 + }, + "pg_stat_database_tup_fetched": { + "iqr": 12089.199999999999, + "max": 52683.13333333333, + "mean": 11575.49090909091, + "median": 5994.866666666667, + "min": 717.6666666666666, + "non_zero_mean": 11575.49090909091, + "non_zero_median": 5994.866666666667, + "percentile25": 3962.733333333333, + "percentile75": 16051.933333333332, + "percentile90": 25135.70666666667, + "percentile99": 50613.906666666655, + "percentile999": 52476.21066666668, + "range": 51965.46666666667, + "samples": 77, + "stdev": 11560.882406749037, + "sum": 891312.7999999998 + }, + "pg_stat_database_tup_inserted": { + "iqr": 28.666666666666668, + "max": 97.2, + "mean": 16.025974025974026, + "median": 5.266666666666667, + "min": 0.0, + "non_zero_mean": 24.68, + "non_zero_median": 17.633333333333333, + "percentile25": 0.0, + "percentile75": 28.666666666666668, + "percentile90": 40.58666666666668, + "percentile99": 90.96799999999996, + "percentile999": 96.57680000000006, + "range": 97.2, + "samples": 77, + "stdev": 22.025257048487493, + "sum": 1234.0000000000007 + }, + "pg_stat_database_tup_returned": { + "iqr": 21007.2, + "max": 85691.86666666667, + "mean": 18427.5012987013, + "median": 10326.0, + "min": 1732.4, + "non_zero_mean": 18427.5012987013, + "non_zero_median": 10326.0, + "percentile25": 6758.733333333334, + "percentile75": 27765.933333333334, + "percentile90": 36225.533333333355, + "percentile99": 80628.54399999997, + "percentile999": 85185.53440000005, + "range": 83959.46666666667, + "samples": 77, + "stdev": 18079.61126854526, + "sum": 1418917.5999999996 + }, + "pg_stat_database_tup_updated": { + "iqr": 31.0, + "max": 76.73333333333333, + "mean": 16.484848484848484, + "median": 6.733333333333333, + "min": 0.0, + "non_zero_mean": 18.396135265700483, + "non_zero_median": 11.4, + "percentile25": 0.26666666666666666, + "percentile75": 31.266666666666666, + "percentile90": 42.4, + "percentile99": 66.8026666666666, + "percentile999": 75.74026666666676, + "range": 76.73333333333333, + "samples": 77, + "stdev": 19.23707019663163, + "sum": 1269.3333333333335 + }, + "pg_stat_database_xact_commit": { + "iqr": 248.4, + "max": 631.4666666666667, + "mean": 153.08744588744588, + "median": 59.4, + "min": 3.8666666666666667, + "non_zero_mean": 153.08744588744588, + "non_zero_median": 59.4, + "percentile25": 9.466666666666667, + "percentile75": 257.8666666666667, + "percentile90": 379.78666666666675, + "percentile99": 601.0159999999997, + "percentile999": 628.4216000000002, + "range": 627.6, + "samples": 77, + "stdev": 168.1767300936692, + "sum": 11787.733333333335 + }, + "pg_stat_database_xact_rollback": { + "iqr": 0.0, + "max": 0.06666666666666667, + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "min": 0.06666666666666667, + "non_zero_mean": 0.06666666666666667, + "non_zero_median": 0.06666666666666667, + "percentile25": 0.06666666666666667, + "percentile75": 0.06666666666666667, + "percentile90": 0.06666666666666667, + "percentile99": 0.06666666666666667, + "percentile999": 0.06666666666666667, + "range": 0.0, + "samples": 77, + "stdev": 0.0, + "sum": 5.133333333333335 + } + }, + "reads_completed": { + "total": { + "iqr": 0.0, + "max": 0.26666666666666666, + "mean": 0.003463203463203463, + "median": 0.0, + "min": 0.0, + "non_zero_mean": 0.26666666666666666, + "non_zero_median": 0.26666666666666666, + "percentile25": 0.0, + "percentile75": 0.0, + "percentile90": 0.0, + "percentile99": 0.06399999999999864, + "percentile999": 0.24640000000000176, + "range": 0.26666666666666666, + "samples": 77, + "stdev": 0.030389487055903455, + "sum": 0.26666666666666666 + } + }, + "writes_completed": { + "total": { + "iqr": 88.93333333333334, + "max": 159.2, + "mean": 57.69437229437229, + "median": 44.0, + "min": 0.4, + "non_zero_mean": 57.69437229437229, + "non_zero_median": 44.0, + "percentile25": 9.466666666666667, + "percentile75": 98.4, + "percentile90": 126.53333333333336, + "percentile99": 156.76799999999997, + "percentile999": 158.95680000000002, + "range": 158.79999999999998, + "samples": 77, + "stdev": 50.18475147669561, + "sum": 4442.466666666667 + } + } + } + }, + "name": "chatty_tasks.yml launching performance test", + "owner": "", + "parameters": { + "ansible_tower": { + "cluster_nodes_count": 2, + "cluster_nodes_list": "gprfc032-vm2.usersys.redhat.com gprfc032-vm3.usersys.redhat.com", + "database_nodes_count": 1, + "database_nodes_list": "gprfc032-vm1.usersys.redhat.com", + "kvm_hosts_list": "gprfc032.sbu.lab.eng.bos.redhat.com", + "version": "automation-controller-4.1.0-2.el8ap.x86_64" + }, + "inventory_id": 3, + "job_ids_str": "61,70,65,60,66,64,63,82,62,74,72,75,73,80,67,78,71,79,69,76,81,83,87,86,84,85,88,89,91,93,95,96,97,99,94,100,101,104,105,112,109,103,110,106,113,114,107,115,108,118,119,122,116,123,121,117,120,124,129,111,125,128,130,132,137,131,134,141,142,138,133,135,145,146,149,139,148,152,150,155,156,151,153,160,159,161,163,147,164,165,167,166,162,168,158,171,169,170,172,173,174,179,176,175,177,183,184,182,178,186,188,192,185,181,187,195,180,194,191,197,200,190,189,201,193,199,203,198,196,202,204,205,206,211,213,207,210,212,215,209,214,208,216,217,220,218,219,222,221,226,224,225,227,229,228,223,232,231,234,236,230,238,239,235,237,233,240,241,243,249,242,245,247,246,250,248,251,257,253,252,260,261,263,255,262,258,259,266,264,268,256,265,267,244,254,269,270,272,271,273,276,274,275,278,277,283,289,280,292,288,286,291,299,293,279,282,285,281,287,297,294,290,284,301,298,295,300,296,309,308,303,307,302,304,305,312,311,306,313,318,317,315,310,322,321,319,314,316,323,328,325,332,326,327,324,320,331,329,330,336,334,337,340,338,339,333,344,343,341,346,348,353,347,342,351,354,358,357,361,362,345,356,355,359,360,363,352,364,369,366,365,335,367,372,370,371,349,350,368,373,376,375,374,379,378,389,384,382,385,377,388,380,390,383,381,386,387,393,396,391,406,400,394,397,395,392,398,399,405,409,403,402,410,404,413,401,412,411,408,407,419,414,418,415,417,416,421,420,422,425,432,423,435,431,429,433,430,434,436,424,428,426,438,437,442,441,447,451,446,443,445,439,444,440,458,452,448,450,453,455,449,463,461,460,457,466,427,456,467,459,464,462,465,469,470,468,472,454,473,471,474,477,475,480,476,478,482,481,479,484,487,493,489,486,483,488,485,491,494,496,490,495,492,498,499,500,507,506,497,504,508,501,509,513,510,516,514,502,503,517,505,515,512,522,518,520,511,521,519,524,523,526,527,529,528,536,530,533,532,534,531,535,537,539,544,538,543,549,540,552,542,547,545,546,553,551,557,555,559,548,558,560,562,550,564,556,563,561,554,566,567,525,568,569,570,541,571,572,565,573", + "job_template_id": 11, + "project_id": 10, + "started": "2022-01-17T21:14:24.770913Z", + "test_chatty_playbook": "chatty_tasks.yml", + "test_chatty_wait": 1800, + "test_launching_concurency": 100, + "test_launching_hosts": 10, + "test_launching_repeat": 5 + }, + "result": "PASS", + "results": { + "created": "2022-01-17T21:14:30.075935Z 2022-01-17T21:14:29.877139Z 2022-01-17T21:14:29.812119Z 2022-01-17T21:14:29.693829Z 2022-01-17T21:14:29.679516Z 2022-01-17T21:14:29.538415Z 2022-01-17T21:14:28.808265Z 2022-01-17T21:14:28.393004Z 2022-01-17T21:14:28.319070Z 2022-01-17T21:14:28.122558Z 2022-01-17T21:14:28.080239Z 2022-01-17T21:14:27.966223Z 2022-01-17T21:14:27.845393Z 2022-01-17T21:14:27.691764Z 2022-01-17T21:14:27.573537Z 2022-01-17T21:14:27.480291Z 2022-01-17T21:14:27.395665Z 2022-01-17T21:14:27.301018Z 2022-01-17T21:14:27.138426Z 2022-01-17T21:14:26.890951Z 2022-01-17T21:14:26.805876Z 2022-01-17T21:14:26.795220Z 2022-01-17T21:14:26.561809Z 2022-01-17T21:14:26.515175Z 2022-01-17T21:14:26.379595Z 2022-01-17T21:14:26.322900Z 2022-01-17T21:14:26.209504Z 2022-01-17T21:14:25.989570Z 2022-01-17T21:14:25.985712Z 2022-01-17T21:14:25.904672Z 2022-01-17T21:14:25.794407Z 2022-01-17T21:14:25.612526Z 2022-01-17T21:14:25.282967Z 2022-01-17T21:14:25.173176Z 2022-01-17T21:14:24.770913Z 2022-01-17T21:14:56.158753Z 2022-01-17T21:14:51.560590Z 2022-01-17T21:14:49.653305Z 2022-01-17T21:14:49.209002Z 2022-01-17T21:14:45.009652Z 2022-01-17T21:14:44.961417Z 2022-01-17T21:14:44.147701Z 2022-01-17T21:14:43.606924Z 2022-01-17T21:14:43.415966Z 2022-01-17T21:14:43.280469Z 2022-01-17T21:14:42.599147Z 2022-01-17T21:14:41.292784Z 2022-01-17T21:14:39.439805Z 2022-01-17T21:14:38.692633Z 2022-01-17T21:14:38.536375Z 2022-01-17T21:14:37.979756Z 2022-01-17T21:14:37.559231Z 2022-01-17T21:14:37.216993Z 2022-01-17T21:14:37.290338Z 2022-01-17T21:14:37.254421Z 2022-01-17T21:14:37.166203Z 2022-01-17T21:14:37.021454Z 2022-01-17T21:14:37.131839Z 2022-01-17T21:14:37.028350Z 2022-01-17T21:14:37.023258Z 2022-01-17T21:14:36.985339Z 2022-01-17T21:14:36.745729Z 2022-01-17T21:14:35.658897Z 2022-01-17T21:14:35.595377Z 2022-01-17T21:14:34.661831Z 2022-01-17T21:14:35.209174Z 2022-01-17T21:14:35.129603Z 2022-01-17T21:14:34.924137Z 2022-01-17T21:14:34.874167Z 2022-01-17T21:14:33.973784Z 2022-01-17T21:14:34.646016Z 2022-01-17T21:14:34.512263Z 2022-01-17T21:14:34.508610Z 2022-01-17T21:14:34.066524Z 2022-01-17T21:14:33.905476Z 2022-01-17T21:14:33.443455Z 2022-01-17T21:14:33.262663Z 2022-01-17T21:14:33.184762Z 2022-01-17T21:14:33.109334Z 2022-01-17T21:14:33.105266Z 2022-01-17T21:14:33.104849Z 2022-01-17T21:14:32.945707Z 2022-01-17T21:14:32.924810Z 2022-01-17T21:14:32.866491Z 2022-01-17T21:14:32.519710Z 2022-01-17T21:14:31.862825Z 2022-01-17T21:14:31.804867Z 2022-01-17T21:14:31.716839Z 2022-01-17T21:14:31.502981Z 2022-01-17T21:14:31.441688Z 2022-01-17T21:14:31.438639Z 2022-01-17T21:14:31.376285Z 2022-01-17T21:14:31.031584Z 2022-01-17T21:14:31.211367Z 2022-01-17T21:14:30.906539Z 2022-01-17T21:14:30.866414Z 2022-01-17T21:14:30.668507Z 2022-01-17T21:14:30.358657Z 2022-01-17T21:14:30.205514Z 2022-01-17T21:14:30.106680Z 2022-01-17T21:19:40.485279Z 2022-01-17T21:19:39.354998Z 2022-01-17T21:19:39.323315Z 2022-01-17T21:19:39.575767Z 2022-01-17T21:19:39.065703Z 2022-01-17T21:19:37.540619Z 2022-01-17T21:19:37.500970Z 2022-01-17T21:19:37.435270Z 2022-01-17T21:19:37.407250Z 2022-01-17T21:19:37.334496Z 2022-01-17T21:19:36.822625Z 2022-01-17T21:19:36.715637Z 2022-01-17T21:19:36.654367Z 2022-01-17T21:19:36.239072Z 2022-01-17T21:19:36.199334Z 2022-01-17T21:19:35.935817Z 2022-01-17T21:19:35.751110Z 2022-01-17T21:19:35.015416Z 2022-01-17T21:19:35.394063Z 2022-01-17T21:19:35.116754Z 2022-01-17T21:19:35.112524Z 2022-01-17T21:19:35.060069Z 2022-01-17T21:19:34.670360Z 2022-01-17T21:19:34.582676Z 2022-01-17T21:19:34.514946Z 2022-01-17T21:19:34.417896Z 2022-01-17T21:19:34.059995Z 2022-01-17T21:19:33.700833Z 2022-01-17T21:19:34.040270Z 2022-01-17T21:19:33.933310Z 2022-01-17T21:19:33.874346Z 2022-01-17T21:19:33.581612Z 2022-01-17T21:19:33.151291Z 2022-01-17T21:19:33.036613Z 2022-01-17T21:19:33.007345Z 2022-01-17T21:19:32.869525Z 2022-01-17T21:19:32.853943Z 2022-01-17T21:19:32.591225Z 2022-01-17T21:19:32.170422Z 2022-01-17T21:19:32.004565Z 2022-01-17T21:19:31.729188Z 2022-01-17T21:19:31.656298Z 2022-01-17T21:19:31.617234Z 2022-01-17T21:19:31.592942Z 2022-01-17T21:19:31.434807Z 2022-01-17T21:19:30.987613Z 2022-01-17T21:19:30.841632Z 2022-01-17T21:19:30.797260Z 2022-01-17T21:19:30.722720Z 2022-01-17T21:19:30.641809Z 2022-01-17T21:19:30.598448Z 2022-01-17T21:19:30.379780Z 2022-01-17T21:19:30.266556Z 2022-01-17T21:19:30.089524Z 2022-01-17T21:19:29.757285Z 2022-01-17T21:19:29.496297Z 2022-01-17T21:19:29.469590Z 2022-01-17T21:19:29.465932Z 2022-01-17T21:19:29.393356Z 2022-01-17T21:19:29.228766Z 2022-01-17T21:19:29.135790Z 2022-01-17T21:19:29.084475Z 2022-01-17T21:19:28.964944Z 2022-01-17T21:19:28.600305Z 2022-01-17T21:19:28.565316Z 2022-01-17T21:19:28.788709Z 2022-01-17T21:19:28.683694Z 2022-01-17T21:19:28.650789Z 2022-01-17T21:19:28.417592Z 2022-01-17T21:19:28.200860Z 2022-01-17T21:19:28.032516Z 2022-01-17T21:19:27.968864Z 2022-01-17T21:19:27.720461Z 2022-01-17T21:19:27.667380Z 2022-01-17T21:19:27.608208Z 2022-01-17T21:19:27.426788Z 2022-01-17T21:19:27.419591Z 2022-01-17T21:19:27.333706Z 2022-01-17T21:19:27.236167Z 2022-01-17T21:19:27.071839Z 2022-01-17T21:19:27.060897Z 2022-01-17T21:19:27.010105Z 2022-01-17T21:19:26.798536Z 2022-01-17T21:19:26.640015Z 2022-01-17T21:19:26.416562Z 2022-01-17T21:19:26.391478Z 2022-01-17T21:19:26.235199Z 2022-01-17T21:19:26.174613Z 2022-01-17T21:19:25.994144Z 2022-01-17T21:19:25.948563Z 2022-01-17T21:19:25.688898Z 2022-01-17T21:19:25.679030Z 2022-01-17T21:19:25.620873Z 2022-01-17T21:19:25.502253Z 2022-01-17T21:19:25.485926Z 2022-01-17T21:19:25.444495Z 2022-01-17T21:19:25.286433Z 2022-01-17T21:19:25.081553Z 2022-01-17T21:19:24.456272Z 2022-01-17T21:19:24.456391Z 2022-01-17T21:23:44.326922Z 2022-01-17T21:23:44.226288Z 2022-01-17T21:23:44.077265Z 2022-01-17T21:23:43.825952Z 2022-01-17T21:23:43.686459Z 2022-01-17T21:23:43.683403Z 2022-01-17T21:23:43.236386Z 2022-01-17T21:23:43.143223Z 2022-01-17T21:23:43.088622Z 2022-01-17T21:23:42.975193Z 2022-01-17T21:23:42.696183Z 2022-01-17T21:23:42.163888Z 2022-01-17T21:23:42.071591Z 2022-01-17T21:23:42.015007Z 2022-01-17T21:23:41.922520Z 2022-01-17T21:23:41.910167Z 2022-01-17T21:23:41.778755Z 2022-01-17T21:23:40.817866Z 2022-01-17T21:23:41.549563Z 2022-01-17T21:23:41.524737Z 2022-01-17T21:23:41.358805Z 2022-01-17T21:23:41.342636Z 2022-01-17T21:23:41.341638Z 2022-01-17T21:23:41.237348Z 2022-01-17T21:23:41.021528Z 2022-01-17T21:23:40.955285Z 2022-01-17T21:23:40.749292Z 2022-01-17T21:23:40.719797Z 2022-01-17T21:23:40.690832Z 2022-01-17T21:23:40.420848Z 2022-01-17T21:23:40.383023Z 2022-01-17T21:23:40.160669Z 2022-01-17T21:23:39.922286Z 2022-01-17T21:23:39.834387Z 2022-01-17T21:23:39.738048Z 2022-01-17T21:23:39.664250Z 2022-01-17T21:23:39.598247Z 2022-01-17T21:23:39.540816Z 2022-01-17T21:23:39.379691Z 2022-01-17T21:23:39.341389Z 2022-01-17T21:23:38.513963Z 2022-01-17T21:23:39.211429Z 2022-01-17T21:23:39.148302Z 2022-01-17T21:23:39.122782Z 2022-01-17T21:23:38.899290Z 2022-01-17T21:23:38.736557Z 2022-01-17T21:23:38.681071Z 2022-01-17T21:23:38.601924Z 2022-01-17T21:23:38.512423Z 2022-01-17T21:23:38.467263Z 2022-01-17T21:23:38.007837Z 2022-01-17T21:23:37.986679Z 2022-01-17T21:23:37.767969Z 2022-01-17T21:23:36.869668Z 2022-01-17T21:23:37.336495Z 2022-01-17T21:23:37.288101Z 2022-01-17T21:23:37.271401Z 2022-01-17T21:23:37.221908Z 2022-01-17T21:23:37.159504Z 2022-01-17T21:23:37.124776Z 2022-01-17T21:23:37.063150Z 2022-01-17T21:23:36.952877Z 2022-01-17T21:23:36.944576Z 2022-01-17T21:23:36.854666Z 2022-01-17T21:23:36.812498Z 2022-01-17T21:23:36.772548Z 2022-01-17T21:23:36.770309Z 2022-01-17T21:23:36.586424Z 2022-01-17T21:23:36.580233Z 2022-01-17T21:23:36.519365Z 2022-01-17T21:23:36.345065Z 2022-01-17T21:23:36.192827Z 2022-01-17T21:23:36.026715Z 2022-01-17T21:23:35.845000Z 2022-01-17T21:23:35.545552Z 2022-01-17T21:23:35.544242Z 2022-01-17T21:23:35.450910Z 2022-01-17T21:23:35.456038Z 2022-01-17T21:23:35.367950Z 2022-01-17T21:23:35.216887Z 2022-01-17T21:23:34.974348Z 2022-01-17T21:23:34.929590Z 2022-01-17T21:23:34.876472Z 2022-01-17T21:23:34.871284Z 2022-01-17T21:23:34.700742Z 2022-01-17T21:23:34.678214Z 2022-01-17T21:23:34.629840Z 2022-01-17T21:23:34.574100Z 2022-01-17T21:23:34.548219Z 2022-01-17T21:23:34.444309Z 2022-01-17T21:23:34.433541Z 2022-01-17T21:23:34.304499Z 2022-01-17T21:23:34.200981Z 2022-01-17T21:23:34.185338Z 2022-01-17T21:23:34.168199Z 2022-01-17T21:23:33.996860Z 2022-01-17T21:23:33.993479Z 2022-01-17T21:23:33.788656Z 2022-01-17T21:23:33.762850Z 2022-01-17T21:23:33.361737Z 2022-01-17T21:27:55.237938Z 2022-01-17T21:27:55.218562Z 2022-01-17T21:27:55.124910Z 2022-01-17T21:27:54.813109Z 2022-01-17T21:27:54.475853Z 2022-01-17T21:27:54.354621Z 2022-01-17T21:27:53.413129Z 2022-01-17T21:27:53.551256Z 2022-01-17T21:27:53.392966Z 2022-01-17T21:27:52.589186Z 2022-01-17T21:27:52.555612Z 2022-01-17T21:27:52.499520Z 2022-01-17T21:27:51.885127Z 2022-01-17T21:27:52.204805Z 2022-01-17T21:27:52.161318Z 2022-01-17T21:27:52.018542Z 2022-01-17T21:27:51.944221Z 2022-01-17T21:27:51.928504Z 2022-01-17T21:27:51.696634Z 2022-01-17T21:27:51.539137Z 2022-01-17T21:27:51.267936Z 2022-01-17T21:27:51.182959Z 2022-01-17T21:27:51.123646Z 2022-01-17T21:27:50.953339Z 2022-01-17T21:27:50.465174Z 2022-01-17T21:27:50.928200Z 2022-01-17T21:27:50.850230Z 2022-01-17T21:27:50.656803Z 2022-01-17T21:27:50.583428Z 2022-01-17T21:27:50.401200Z 2022-01-17T21:27:50.375109Z 2022-01-17T21:27:50.324212Z 2022-01-17T21:27:50.138186Z 2022-01-17T21:27:50.033008Z 2022-01-17T21:27:49.873622Z 2022-01-17T21:27:49.744661Z 2022-01-17T21:27:49.543059Z 2022-01-17T21:27:49.241812Z 2022-01-17T21:27:49.126180Z 2022-01-17T21:27:49.100340Z 2022-01-17T21:27:49.073996Z 2022-01-17T21:27:49.062899Z 2022-01-17T21:27:48.773260Z 2022-01-17T21:27:48.665147Z 2022-01-17T21:27:48.616666Z 2022-01-17T21:27:48.589165Z 2022-01-17T21:27:48.568945Z 2022-01-17T21:27:48.496211Z 2022-01-17T21:27:48.463153Z 2022-01-17T21:27:48.427211Z 2022-01-17T21:27:48.147413Z 2022-01-17T21:27:47.887190Z 2022-01-17T21:27:47.609656Z 2022-01-17T21:27:47.526815Z 2022-01-17T21:27:47.410160Z 2022-01-17T21:27:47.413239Z 2022-01-17T21:27:47.348834Z 2022-01-17T21:27:47.011989Z 2022-01-17T21:27:46.998324Z 2022-01-17T21:27:46.659539Z 2022-01-17T21:27:46.609422Z 2022-01-17T21:27:46.575019Z 2022-01-17T21:27:46.553227Z 2022-01-17T21:27:46.488767Z 2022-01-17T21:27:46.459164Z 2022-01-17T21:27:46.407372Z 2022-01-17T21:27:46.305834Z 2022-01-17T21:27:45.812929Z 2022-01-17T21:27:45.818713Z 2022-01-17T21:27:45.634507Z 2022-01-17T21:27:45.624124Z 2022-01-17T21:27:45.545763Z 2022-01-17T21:27:45.518209Z 2022-01-17T21:27:45.491262Z 2022-01-17T21:27:45.485195Z 2022-01-17T21:27:45.243290Z 2022-01-17T21:27:45.165894Z 2022-01-17T21:27:45.117545Z 2022-01-17T21:27:45.046772Z 2022-01-17T21:27:44.945943Z 2022-01-17T21:27:44.704667Z 2022-01-17T21:27:44.703715Z 2022-01-17T21:27:44.641598Z 2022-01-17T21:27:44.633242Z 2022-01-17T21:27:44.534926Z 2022-01-17T21:27:44.445533Z 2022-01-17T21:27:44.406281Z 2022-01-17T21:27:44.269886Z 2022-01-17T21:27:44.251554Z 2022-01-17T21:27:44.247597Z 2022-01-17T21:27:44.227293Z 2022-01-17T21:27:43.888741Z 2022-01-17T21:27:43.782687Z 2022-01-17T21:27:43.743860Z 2022-01-17T21:27:43.406924Z 2022-01-17T21:27:43.170770Z 2022-01-17T21:27:43.054567Z 2022-01-17T21:27:42.646905Z 2022-01-17T21:27:42.578922Z 2022-01-17T21:27:42.449774Z 2022-01-17T21:32:03.363956Z 2022-01-17T21:32:02.787744Z 2022-01-17T21:32:02.798862Z 2022-01-17T21:32:02.483960Z 2022-01-17T21:32:02.426011Z 2022-01-17T21:32:01.994488Z 2022-01-17T21:32:02.196063Z 2022-01-17T21:32:01.556084Z 2022-01-17T21:32:01.796652Z 2022-01-17T21:32:01.628336Z 2022-01-17T21:32:01.522995Z 2022-01-17T21:32:01.222314Z 2022-01-17T21:32:01.136618Z 2022-01-17T21:32:00.977378Z 2022-01-17T21:32:00.735619Z 2022-01-17T21:32:00.704089Z 2022-01-17T21:32:00.258258Z 2022-01-17T21:32:00.168913Z 2022-01-17T21:32:00.146418Z 2022-01-17T21:32:00.025436Z 2022-01-17T21:31:59.991647Z 2022-01-17T21:31:59.847119Z 2022-01-17T21:31:59.766036Z 2022-01-17T21:31:59.513925Z 2022-01-17T21:31:59.505941Z 2022-01-17T21:31:59.518180Z 2022-01-17T21:31:59.320350Z 2022-01-17T21:31:59.280392Z 2022-01-17T21:31:59.153495Z 2022-01-17T21:31:59.146070Z 2022-01-17T21:31:59.026717Z 2022-01-17T21:31:58.930248Z 2022-01-17T21:31:58.878552Z 2022-01-17T21:31:58.357498Z 2022-01-17T21:31:58.266235Z 2022-01-17T21:31:58.195604Z 2022-01-17T21:31:58.004757Z 2022-01-17T21:31:57.622108Z 2022-01-17T21:31:57.420584Z 2022-01-17T21:31:57.352723Z 2022-01-17T21:31:57.289662Z 2022-01-17T21:31:57.289722Z 2022-01-17T21:31:57.250480Z 2022-01-17T21:31:57.129280Z 2022-01-17T21:31:57.062042Z 2022-01-17T21:31:56.886364Z 2022-01-17T21:31:56.877609Z 2022-01-17T21:31:56.675609Z 2022-01-17T21:31:56.418530Z 2022-01-17T21:31:56.163888Z 2022-01-17T21:31:56.152924Z 2022-01-17T21:31:56.133571Z 2022-01-17T21:31:55.956131Z 2022-01-17T21:31:55.817511Z 2022-01-17T21:31:55.812337Z 2022-01-17T21:31:55.767562Z 2022-01-17T21:31:55.504173Z 2022-01-17T21:31:55.475756Z 2022-01-17T21:31:55.325943Z 2022-01-17T21:31:55.298211Z 2022-01-17T21:31:54.781980Z 2022-01-17T21:31:54.747323Z 2022-01-17T21:31:54.707407Z 2022-01-17T21:31:54.666754Z 2022-01-17T21:31:54.627237Z 2022-01-17T21:31:54.624477Z 2022-01-17T21:31:54.527345Z 2022-01-17T21:31:54.517012Z 2022-01-17T21:31:54.510019Z 2022-01-17T21:31:54.427087Z 2022-01-17T21:31:54.326758Z 2022-01-17T21:31:54.088692Z 2022-01-17T21:31:53.857474Z 2022-01-17T21:31:53.824654Z 2022-01-17T21:31:53.796577Z 2022-01-17T21:31:53.461080Z 2022-01-17T21:31:53.460012Z 2022-01-17T21:31:53.357293Z 2022-01-17T21:31:53.350029Z 2022-01-17T21:31:53.070850Z 2022-01-17T21:31:53.029094Z 2022-01-17T21:31:53.018495Z 2022-01-17T21:31:52.932925Z 2022-01-17T21:31:52.859341Z 2022-01-17T21:31:52.720100Z 2022-01-17T21:31:52.369462Z 2022-01-17T21:31:52.332195Z 2022-01-17T21:31:52.310274Z 2022-01-17T21:31:52.242848Z 2022-01-17T21:31:52.112222Z 2022-01-17T21:31:52.009743Z 2022-01-17T21:31:51.997124Z 2022-01-17T21:31:51.773111Z 2022-01-17T21:31:51.655359Z 2022-01-17T21:31:51.586803Z 2022-01-17T21:31:51.186823Z 2022-01-17T21:31:51.067004Z 2022-01-17T21:31:50.902369Z 2022-01-17T21:31:50.663165Z 2022-01-17T21:31:50.504947Z", + "created_diff": 16.805841, + "ended": "2022-01-17T21:33:34.060942Z", + "error_job_ids": " []", + "event_first": "2022-01-17T21:15:43.187562Z 2022-01-17T21:15:16.547187Z 2022-01-17T21:15:48.102757Z 2022-01-17T21:15:02.688494Z 2022-01-17T21:15:45.849790Z 2022-01-17T21:15:08.769078Z 2022-01-17T21:15:41.323147Z 2022-01-17T21:15:58.713848Z 2022-01-17T21:15:43.262663Z 2022-01-17T21:15:50.036110Z 2022-01-17T21:15:35.325686Z 2022-01-17T21:16:42.764132Z 2022-01-17T21:15:18.010350Z 2022-01-17T21:16:25.537759Z 2022-01-17T21:15:26.031034Z 2022-01-17T21:16:34.337532Z 2022-01-17T21:15:27.810623Z 2022-01-17T21:17:11.458406Z 2022-01-17T21:15:08.466951Z 2022-01-17T21:15:37.831172Z 2022-01-17T21:15:05.432689Z 2022-01-17T21:16:52.449772Z 2022-01-17T21:14:58.140512Z 2022-01-17T21:17:31.714753Z 2022-01-17T21:14:48.583362Z 2022-01-17T21:17:21.783191Z 2022-01-17T21:15:00.081651Z 2022-01-17T21:16:07.919233Z 2022-01-17T21:14:51.263456Z 2022-01-17T21:16:17.191675Z 2022-01-17T21:14:43.089575Z 2022-01-17T21:17:02.655794Z 2022-01-17T21:14:39.027140Z 2022-01-17T21:14:47.996996Z 2022-01-17T21:14:35.481428Z 2022-01-17T21:16:16.752150Z 2022-01-17T21:15:45.368712Z 2022-01-17T21:16:15.175169Z 2022-01-17T21:16:16.324522Z 2022-01-17T21:16:17.499238Z 2022-01-17T21:15:43.592100Z 2022-01-17T21:16:11.460447Z 2022-01-17T21:15:42.698362Z 2022-01-17T21:16:13.952505Z 2022-01-17T21:15:44.776368Z 2022-01-17T21:16:11.203635Z 2022-01-17T21:15:41.769212Z 2022-01-17T21:16:15.848675Z 2022-01-17T21:15:43.337321Z 2022-01-17T21:16:11.201979Z 2022-01-17T21:15:44.024073Z 2022-01-17T21:16:11.990634Z 2022-01-17T21:15:40.998241Z 2022-01-17T21:15:40.223947Z 2022-01-17T21:16:12.496885Z 2022-01-17T21:16:13.430961Z 2022-01-17T21:16:12.905492Z 2022-01-17T21:15:40.727357Z 2022-01-17T21:16:02.611438Z 2022-01-17T21:16:03.089199Z 2022-01-17T21:16:10.725761Z 2022-01-17T21:15:34.921127Z 2022-01-17T21:15:59.814915Z 2022-01-17T21:15:28.805719Z 2022-01-17T21:16:05.717616Z 2022-01-17T21:16:07.723449Z 2022-01-17T21:15:30.921595Z 2022-01-17T21:16:03.976805Z 2022-01-17T21:15:28.478057Z 2022-01-17T21:15:27.790237Z 2022-01-17T21:15:36.168182Z 2022-01-17T21:16:03.223027Z 2022-01-17T21:15:22.123339Z 2022-01-17T21:16:06.826029Z 2022-01-17T21:16:06.468781Z 2022-01-17T21:15:29.448375Z 2022-01-17T21:16:05.213532Z 2022-01-17T21:15:18.696248Z 2022-01-17T21:15:55.962616Z 2022-01-17T21:15:24.475248Z 2022-01-17T21:15:57.127725Z 2022-01-17T21:15:24.982328Z 2022-01-17T21:16:04.184174Z 2022-01-17T21:15:29.949441Z 2022-01-17T21:15:58.869156Z 2022-01-17T21:15:26.766083Z 2022-01-17T21:15:54.210054Z 2022-01-17T21:15:19.483424Z 2022-01-17T21:16:00.342579Z 2022-01-17T21:15:20.218167Z 2022-01-17T21:15:50.878622Z 2022-01-17T21:15:21.334962Z 2022-01-17T21:15:10.471450Z 2022-01-17T21:15:50.128729Z 2022-01-17T21:15:57.662594Z 2022-01-17T21:15:10.223240Z 2022-01-17T21:15:55.217776Z 2022-01-17T21:15:07.484310Z 2022-01-17T21:15:49.650785Z 2022-01-17T21:15:13.794996Z 2022-01-17T21:21:06.696492Z 2022-01-17T21:21:08.379726Z 2022-01-17T21:20:58.999339Z 2022-01-17T21:21:01.219451Z 2022-01-17T21:21:06.696492Z 2022-01-17T21:21:01.470073Z 2022-01-17T21:21:08.029551Z 2022-01-17T21:21:00.433221Z 2022-01-17T21:21:06.462525Z 2022-01-17T21:20:59.205407Z 2022-01-17T21:21:05.217423Z 2022-01-17T21:21:01.219451Z 2022-01-17T21:21:05.192478Z 2022-01-17T21:20:56.933506Z 2022-01-17T21:20:59.682458Z 2022-01-17T21:20:55.064145Z 2022-01-17T21:21:00.186129Z 2022-01-17T21:21:01.766194Z 2022-01-17T21:20:53.677594Z 2022-01-17T21:21:07.617496Z 2022-01-17T21:21:01.220998Z 2022-01-17T21:20:56.581078Z 2022-01-17T21:20:52.919341Z 2022-01-17T21:20:59.351056Z 2022-01-17T21:20:48.539570Z 2022-01-17T21:20:57.891581Z 2022-01-17T21:20:53.925458Z 2022-01-17T21:20:56.269170Z 2022-01-17T21:20:59.104707Z 2022-01-17T21:21:05.944879Z 2022-01-17T21:20:53.664490Z 2022-01-17T21:20:44.246313Z 2022-01-17T21:21:01.519049Z 2022-01-17T21:20:50.849416Z 2022-01-17T21:21:00.354633Z 2022-01-17T21:20:54.944977Z 2022-01-17T21:20:47.938548Z 2022-01-17T21:20:53.702641Z 2022-01-17T21:20:51.249578Z 2022-01-17T21:20:50.090062Z 2022-01-17T21:20:48.200574Z 2022-01-17T21:20:49.839565Z 2022-01-17T21:20:52.672298Z 2022-01-17T21:20:47.781132Z 2022-01-17T21:20:53.591818Z 2022-01-17T21:20:54.944977Z 2022-01-17T21:20:59.682458Z 2022-01-17T21:20:44.045054Z 2022-01-17T21:20:56.224382Z 2022-01-17T21:20:43.850268Z 2022-01-17T21:20:50.970628Z 2022-01-17T21:20:39.496179Z 2022-01-17T21:20:47.864182Z 2022-01-17T21:20:40.845524Z 2022-01-17T21:20:49.162481Z 2022-01-17T21:20:43.417220Z 2022-01-17T21:20:46.929864Z 2022-01-17T21:20:37.481253Z 2022-01-17T21:20:37.594585Z 2022-01-17T21:20:43.157421Z 2022-01-17T21:20:40.006352Z 2022-01-17T21:20:38.150083Z 2022-01-17T21:20:37.334644Z 2022-01-17T21:20:36.314664Z 2022-01-17T21:20:34.743274Z 2022-01-17T21:20:34.477288Z 2022-01-17T21:20:40.519021Z 2022-01-17T21:20:43.850268Z 2022-01-17T21:20:37.154513Z 2022-01-17T21:20:34.423310Z 2022-01-17T21:20:28.608374Z 2022-01-17T21:20:34.749551Z 2022-01-17T21:20:28.441865Z 2022-01-17T21:20:18.032198Z 2022-01-17T21:20:26.586073Z 2022-01-17T21:20:25.727946Z 2022-01-17T21:20:26.140169Z 2022-01-17T21:20:14.733887Z 2022-01-17T21:20:13.999625Z 2022-01-17T21:20:22.384937Z 2022-01-17T21:20:03.924998Z 2022-01-17T21:20:20.144572Z 2022-01-17T21:19:50.161164Z 2022-01-17T21:19:50.147622Z 2022-01-17T21:19:52.614698Z 2022-01-17T21:19:50.905509Z 2022-01-17T21:19:49.648064Z 2022-01-17T21:19:48.508695Z 2022-01-17T21:19:48.889414Z 2022-01-17T21:19:47.224435Z 2022-01-17T21:19:45.300876Z 2022-01-17T21:19:45.454009Z 2022-01-17T21:19:43.740973Z 2022-01-17T21:19:46.984685Z 2022-01-17T21:19:38.181861Z 2022-01-17T21:19:44.233620Z 2022-01-17T21:19:34.927212Z 2022-01-17T21:19:40.083676Z 2022-01-17T21:19:35.315344Z 2022-01-17T21:19:30.992821Z 2022-01-17T21:25:11.727026Z 2022-01-17T21:25:08.196285Z 2022-01-17T21:25:10.722612Z 2022-01-17T21:25:07.286386Z 2022-01-17T21:25:16.124532Z 2022-01-17T21:25:07.974070Z 2022-01-17T21:25:13.351334Z 2022-01-17T21:25:09.328082Z 2022-01-17T21:25:15.425827Z 2022-01-17T21:25:07.202534Z 2022-01-17T21:25:14.456409Z 2022-01-17T21:25:04.813426Z 2022-01-17T21:25:16.879681Z 2022-01-17T21:25:04.317059Z 2022-01-17T21:25:15.929598Z 2022-01-17T21:25:01.350596Z 2022-01-17T21:25:12.202677Z 2022-01-17T21:25:03.562053Z 2022-01-17T21:25:07.976217Z 2022-01-17T21:25:14.077271Z 2022-01-17T21:25:02.944306Z 2022-01-17T21:25:12.202677Z 2022-01-17T21:25:04.521067Z 2022-01-17T21:25:12.607557Z 2022-01-17T21:25:02.127614Z 2022-01-17T21:25:15.202649Z 2022-01-17T21:25:09.445759Z 2022-01-17T21:25:00.840159Z 2022-01-17T21:25:06.568366Z 2022-01-17T21:24:56.993580Z 2022-01-17T21:25:11.986507Z 2022-01-17T21:25:02.320265Z 2022-01-17T21:25:06.821274Z 2022-01-17T21:25:01.851676Z 2022-01-17T21:25:08.118478Z 2022-01-17T21:24:53.683788Z 2022-01-17T21:25:05.560375Z 2022-01-17T21:24:59.037507Z 2022-01-17T21:25:08.558439Z 2022-01-17T21:24:59.286158Z 2022-01-17T21:24:54.711295Z 2022-01-17T21:24:59.143179Z 2022-01-17T21:24:56.234931Z 2022-01-17T21:25:11.214133Z 2022-01-17T21:24:50.654310Z 2022-01-17T21:24:59.336243Z 2022-01-17T21:24:52.745499Z 2022-01-17T21:25:01.251579Z 2022-01-17T21:24:57.630528Z 2022-01-17T21:24:48.062088Z 2022-01-17T21:24:54.960739Z 2022-01-17T21:24:47.806484Z 2022-01-17T21:25:02.621705Z 2022-01-17T21:24:45.592830Z 2022-01-17T21:24:47.304124Z 2022-01-17T21:24:55.911633Z 2022-01-17T21:24:47.052267Z 2022-01-17T21:24:55.925011Z 2022-01-17T21:24:43.986446Z 2022-01-17T21:24:58.734565Z 2022-01-17T21:24:52.495532Z 2022-01-17T21:24:58.896512Z 2022-01-17T21:24:39.495194Z 2022-01-17T21:24:46.565104Z 2022-01-17T21:24:57.630528Z 2022-01-17T21:24:42.332288Z 2022-01-17T21:24:55.034955Z 2022-01-17T21:24:36.249643Z 2022-01-17T21:24:53.362267Z 2022-01-17T21:24:47.094115Z 2022-01-17T21:24:15.063113Z 2022-01-17T21:24:18.908789Z 2022-01-17T21:24:14.290165Z 2022-01-17T21:24:20.323332Z 2022-01-17T21:24:19.334918Z 2022-01-17T21:24:11.368649Z 2022-01-17T21:24:14.295353Z 2022-01-17T21:24:07.369587Z 2022-01-17T21:24:08.039512Z 2022-01-17T21:24:16.573068Z 2022-01-17T21:24:11.464526Z 2022-01-17T21:24:10.520522Z 2022-01-17T21:24:04.968373Z 2022-01-17T21:24:08.658717Z 2022-01-17T21:24:04.192337Z 2022-01-17T21:24:09.043241Z 2022-01-17T21:24:02.743907Z 2022-01-17T21:24:08.683672Z 2022-01-17T21:23:59.372190Z 2022-01-17T21:24:08.151923Z 2022-01-17T21:23:59.150413Z 2022-01-17T21:24:04.336647Z 2022-01-17T21:23:57.857085Z 2022-01-17T21:24:04.588025Z 2022-01-17T21:23:56.560537Z 2022-01-17T21:23:48.083801Z 2022-01-17T21:23:45.390154Z 2022-01-17T21:23:48.332525Z 2022-01-17T21:23:44.062571Z 2022-01-17T21:23:42.395578Z 2022-01-17T21:29:25.707515Z 2022-01-17T21:29:19.798570Z 2022-01-17T21:29:21.975797Z 2022-01-17T21:29:19.186184Z 2022-01-17T21:29:22.628495Z 2022-01-17T21:29:19.944264Z 2022-01-17T21:29:21.436416Z 2022-01-17T21:29:25.086953Z 2022-01-17T21:29:23.630459Z 2022-01-17T21:29:21.751318Z 2022-01-17T21:29:25.086953Z 2022-01-17T21:29:20.689469Z 2022-01-17T21:29:20.723805Z 2022-01-17T21:29:23.630459Z 2022-01-17T21:29:20.980250Z 2022-01-17T21:29:14.508672Z 2022-01-17T21:29:24.656460Z 2022-01-17T21:29:20.430250Z 2022-01-17T21:29:22.527288Z 2022-01-17T21:29:19.186184Z 2022-01-17T21:29:09.915449Z 2022-01-17T21:29:14.276914Z 2022-01-17T21:29:14.378356Z 2022-01-17T21:29:21.975797Z 2022-01-17T21:29:14.129144Z 2022-01-17T21:29:17.418065Z 2022-01-17T21:29:20.973626Z 2022-01-17T21:29:09.806228Z 2022-01-17T21:29:14.617537Z 2022-01-17T21:29:19.976619Z 2022-01-17T21:29:13.252150Z 2022-01-17T21:29:13.085487Z 2022-01-17T21:29:12.748081Z 2022-01-17T21:29:19.819727Z 2022-01-17T21:29:07.770341Z 2022-01-17T21:29:16.870052Z 2022-01-17T21:29:07.386413Z 2022-01-17T21:29:09.499570Z 2022-01-17T21:29:13.472107Z 2022-01-17T21:29:13.336009Z 2022-01-17T21:29:07.789730Z 2022-01-17T21:29:13.218783Z 2022-01-17T21:29:08.791095Z 2022-01-17T21:29:07.749711Z 2022-01-17T21:29:05.361366Z 2022-01-17T21:29:09.008405Z 2022-01-17T21:28:59.566584Z 2022-01-17T21:29:00.272428Z 2022-01-17T21:29:03.572115Z 2022-01-17T21:29:06.239973Z 2022-01-17T21:29:05.860072Z 2022-01-17T21:29:05.646418Z 2022-01-17T21:29:04.494258Z 2022-01-17T21:29:09.914133Z 2022-01-17T21:29:09.912696Z 2022-01-17T21:28:56.108087Z 2022-01-17T21:29:11.997508Z 2022-01-17T21:29:03.350589Z 2022-01-17T21:28:58.147316Z 2022-01-17T21:29:05.324452Z 2022-01-17T21:28:51.510285Z 2022-01-17T21:28:59.594578Z 2022-01-17T21:28:55.669218Z 2022-01-17T21:29:01.169711Z 2022-01-17T21:28:56.377341Z 2022-01-17T21:29:01.851539Z 2022-01-17T21:28:58.236303Z 2022-01-17T21:28:57.306070Z 2022-01-17T21:28:56.114901Z 2022-01-17T21:28:52.360196Z 2022-01-17T21:28:52.450184Z 2022-01-17T21:28:56.055555Z 2022-01-17T21:28:40.527236Z 2022-01-17T21:28:59.361492Z 2022-01-17T21:28:49.955116Z 2022-01-17T21:28:56.763858Z 2022-01-17T21:28:44.720711Z 2022-01-17T21:28:56.941567Z 2022-01-17T21:28:27.967337Z 2022-01-17T21:28:43.930630Z 2022-01-17T21:28:36.352297Z 2022-01-17T21:28:41.233787Z 2022-01-17T21:28:35.820837Z 2022-01-17T21:28:39.289562Z 2022-01-17T21:28:35.549021Z 2022-01-17T21:28:28.650001Z 2022-01-17T21:28:32.584281Z 2022-01-17T21:28:16.181225Z 2022-01-17T21:28:02.273200Z 2022-01-17T21:28:17.838984Z 2022-01-17T21:28:01.933155Z 2022-01-17T21:28:18.848476Z 2022-01-17T21:28:00.223651Z 2022-01-17T21:28:08.809334Z 2022-01-17T21:27:59.705757Z 2022-01-17T21:27:57.864386Z 2022-01-17T21:27:55.851202Z 2022-01-17T21:27:55.484624Z 2022-01-17T21:27:50.105305Z 2022-01-17T21:27:53.115187Z 2022-01-17T21:33:28.918064Z 2022-01-17T21:33:22.630479Z 2022-01-17T21:33:25.449938Z 2022-01-17T21:33:28.610023Z 2022-01-17T21:33:23.602245Z 2022-01-17T21:33:27.726576Z 2022-01-17T21:33:31.551447Z 2022-01-17T21:33:29.887623Z 2022-01-17T21:33:32.075831Z 2022-01-17T21:33:27.728440Z 2022-01-17T21:33:26.963210Z 2022-01-17T21:33:26.945425Z 2022-01-17T21:33:23.390140Z 2022-01-17T21:33:31.300449Z 2022-01-17T21:33:24.697072Z 2022-01-17T21:33:31.060592Z 2022-01-17T21:33:26.206285Z 2022-01-17T21:33:29.632212Z 2022-01-17T21:33:24.645364Z 2022-01-17T21:33:25.732958Z 2022-01-17T21:33:21.875546Z 2022-01-17T21:33:32.321408Z 2022-01-17T21:33:20.381612Z 2022-01-17T21:33:21.895001Z 2022-01-17T21:33:22.027932Z 2022-01-17T21:33:32.075831Z 2022-01-17T21:33:23.602245Z 2022-01-17T21:33:28.817909Z 2022-01-17T21:33:21.071238Z 2022-01-17T21:33:24.690825Z 2022-01-17T21:33:15.385182Z 2022-01-17T21:33:22.456752Z 2022-01-17T21:33:21.119070Z 2022-01-17T21:33:22.709541Z 2022-01-17T21:33:19.922376Z 2022-01-17T21:33:25.732958Z 2022-01-17T21:33:14.437568Z 2022-01-17T21:33:25.449938Z 2022-01-17T21:33:19.172894Z 2022-01-17T21:33:22.709541Z 2022-01-17T21:33:19.257050Z 2022-01-17T21:33:19.169187Z 2022-01-17T21:33:15.345276Z 2022-01-17T21:33:23.292589Z 2022-01-17T21:33:17.042067Z 2022-01-17T21:33:13.787562Z 2022-01-17T21:33:07.907538Z 2022-01-17T21:33:16.580799Z 2022-01-17T21:33:12.089194Z 2022-01-17T21:33:21.274397Z 2022-01-17T21:33:08.247591Z 2022-01-17T21:33:12.481070Z 2022-01-17T21:33:12.711741Z 2022-01-17T21:33:19.759848Z 2022-01-17T21:33:13.855013Z 2022-01-17T21:33:16.063346Z 2022-01-17T21:33:06.057793Z 2022-01-17T21:33:09.977167Z 2022-01-17T21:33:07.238764Z 2022-01-17T21:33:10.484835Z 2022-01-17T21:33:14.947211Z 2022-01-17T21:32:51.169169Z 2022-01-17T21:33:17.542507Z 2022-01-17T21:32:49.111979Z 2022-01-17T21:33:02.193778Z 2022-01-17T21:32:45.447951Z 2022-01-17T21:32:58.991474Z 2022-01-17T21:32:38.549246Z 2022-01-17T21:33:11.956703Z 2022-01-17T21:32:52.009218Z 2022-01-17T21:32:37.263922Z 2022-01-17T21:32:35.580479Z 2022-01-17T21:32:36.493234Z 2022-01-17T21:32:46.958846Z 2022-01-17T21:32:28.409398Z 2022-01-17T21:32:53.655777Z 2022-01-17T21:32:34.825591Z 2022-01-17T21:32:35.950558Z 2022-01-17T21:32:30.294485Z 2022-01-17T21:32:34.552789Z 2022-01-17T21:32:27.999457Z 2022-01-17T21:32:32.544148Z 2022-01-17T21:32:25.986284Z 2022-01-17T21:32:23.475633Z 2022-01-17T21:32:24.957196Z 2022-01-17T21:32:33.904802Z 2022-01-17T21:32:21.191225Z 2022-01-17T21:32:22.699322Z 2022-01-17T21:32:22.546158Z 2022-01-17T21:32:24.092186Z 2022-01-17T21:32:19.570670Z 2022-01-17T21:32:14.366560Z 2022-01-17T21:32:07.916244Z 2022-01-17T21:32:10.213520Z 2022-01-17T21:32:05.639945Z 2022-01-17T21:32:05.305841Z 2022-01-17T21:32:02.904021Z 2022-01-17T21:32:04.570576Z 2022-01-17T21:31:58.901173Z 2022-01-17T21:31:58.931844Z", + "event_last": "2022-01-17T21:15:48.549148Z 2022-01-17T21:15:20.141919Z 2022-01-17T21:15:52.274323Z 2022-01-17T21:15:05.789917Z 2022-01-17T21:15:51.260600Z 2022-01-17T21:15:13.175478Z 2022-01-17T21:15:46.291190Z 2022-01-17T21:15:58.880663Z 2022-01-17T21:15:48.347828Z 2022-01-17T21:15:50.544870Z 2022-01-17T21:15:39.947291Z 2022-01-17T21:16:44.123115Z 2022-01-17T21:15:23.654279Z 2022-01-17T21:16:28.253918Z 2022-01-17T21:15:31.505780Z 2022-01-17T21:16:36.578189Z 2022-01-17T21:15:33.343281Z 2022-01-17T21:17:11.929761Z 2022-01-17T21:15:14.150026Z 2022-01-17T21:15:39.831906Z 2022-01-17T21:15:10.622255Z 2022-01-17T21:16:52.850781Z 2022-01-17T21:15:02.371313Z 2022-01-17T21:17:32.127205Z 2022-01-17T21:14:51.427372Z 2022-01-17T21:17:22.201088Z 2022-01-17T21:15:03.672333Z 2022-01-17T21:16:08.585024Z 2022-01-17T21:14:54.560278Z 2022-01-17T21:16:17.710711Z 2022-01-17T21:14:46.207301Z 2022-01-17T21:17:03.025784Z 2022-01-17T21:14:41.174376Z 2022-01-17T21:14:50.139894Z 2022-01-17T21:14:37.274289Z 2022-01-17T21:16:17.975847Z 2022-01-17T21:15:45.840239Z 2022-01-17T21:16:17.148252Z 2022-01-17T21:16:18.031361Z 2022-01-17T21:16:20.625984Z 2022-01-17T21:15:44.780367Z 2022-01-17T21:16:13.527277Z 2022-01-17T21:15:43.911740Z 2022-01-17T21:16:16.728495Z 2022-01-17T21:15:45.562342Z 2022-01-17T21:16:14.136804Z 2022-01-17T21:15:43.068033Z 2022-01-17T21:16:17.586054Z 2022-01-17T21:15:44.287303Z 2022-01-17T21:16:14.290622Z 2022-01-17T21:15:45.107154Z 2022-01-17T21:16:14.989510Z 2022-01-17T21:15:42.027004Z 2022-01-17T21:15:41.982278Z 2022-01-17T21:16:14.856369Z 2022-01-17T21:16:16.262436Z 2022-01-17T21:16:15.833800Z 2022-01-17T21:15:42.147797Z 2022-01-17T21:16:07.614207Z 2022-01-17T21:16:07.701313Z 2022-01-17T21:16:12.776636Z 2022-01-17T21:15:36.811929Z 2022-01-17T21:16:04.843885Z 2022-01-17T21:15:32.083845Z 2022-01-17T21:16:08.776049Z 2022-01-17T21:16:10.299266Z 2022-01-17T21:15:33.478528Z 2022-01-17T21:16:07.464340Z 2022-01-17T21:15:31.539587Z 2022-01-17T21:15:31.000350Z 2022-01-17T21:15:38.304899Z 2022-01-17T21:16:08.646370Z 2022-01-17T21:15:25.825934Z 2022-01-17T21:16:10.684310Z 2022-01-17T21:16:11.270147Z 2022-01-17T21:15:32.232727Z 2022-01-17T21:16:08.424066Z 2022-01-17T21:15:21.649051Z 2022-01-17T21:16:00.452907Z 2022-01-17T21:15:28.385086Z 2022-01-17T21:16:01.759670Z 2022-01-17T21:15:27.322923Z 2022-01-17T21:16:07.521962Z 2022-01-17T21:15:32.684545Z 2022-01-17T21:16:03.689914Z 2022-01-17T21:15:30.358729Z 2022-01-17T21:15:59.434310Z 2022-01-17T21:15:23.724637Z 2022-01-17T21:16:05.924797Z 2022-01-17T21:15:23.870517Z 2022-01-17T21:15:55.227463Z 2022-01-17T21:15:25.062977Z 2022-01-17T21:15:14.034085Z 2022-01-17T21:15:55.216343Z 2022-01-17T21:16:03.014896Z 2022-01-17T21:15:13.175478Z 2022-01-17T21:15:59.034301Z 2022-01-17T21:15:10.082040Z 2022-01-17T21:15:53.726290Z 2022-01-17T21:15:18.152805Z 2022-01-17T21:21:08.464403Z 2022-01-17T21:21:09.384411Z 2022-01-17T21:21:00.033117Z 2022-01-17T21:21:01.785297Z 2022-01-17T21:21:08.398311Z 2022-01-17T21:21:02.354413Z 2022-01-17T21:21:09.223492Z 2022-01-17T21:21:01.356907Z 2022-01-17T21:21:08.111847Z 2022-01-17T21:21:00.229322Z 2022-01-17T21:21:07.026280Z 2022-01-17T21:21:02.114032Z 2022-01-17T21:21:07.096498Z 2022-01-17T21:20:59.198075Z 2022-01-17T21:21:03.073407Z 2022-01-17T21:20:57.949240Z 2022-01-17T21:21:03.526914Z 2022-01-17T21:21:04.014306Z 2022-01-17T21:20:56.365917Z 2022-01-17T21:21:08.940172Z 2022-01-17T21:21:04.472367Z 2022-01-17T21:20:58.722457Z 2022-01-17T21:20:56.231450Z 2022-01-17T21:21:01.935511Z 2022-01-17T21:20:51.520900Z 2022-01-17T21:21:01.290687Z 2022-01-17T21:20:56.576923Z 2022-01-17T21:20:59.399756Z 2022-01-17T21:21:02.852291Z 2022-01-17T21:21:07.162751Z 2022-01-17T21:20:56.525569Z 2022-01-17T21:20:48.177560Z 2022-01-17T21:21:04.141370Z 2022-01-17T21:20:52.999917Z 2022-01-17T21:21:03.165396Z 2022-01-17T21:20:57.870865Z 2022-01-17T21:20:50.774900Z 2022-01-17T21:20:56.609778Z 2022-01-17T21:20:55.336290Z 2022-01-17T21:20:52.277916Z 2022-01-17T21:20:51.998269Z 2022-01-17T21:20:51.912946Z 2022-01-17T21:20:56.021425Z 2022-01-17T21:20:50.417242Z 2022-01-17T21:20:55.856317Z 2022-01-17T21:20:57.471255Z 2022-01-17T21:21:02.997884Z 2022-01-17T21:20:48.938309Z 2022-01-17T21:20:59.294787Z 2022-01-17T21:20:48.155978Z 2022-01-17T21:20:55.324560Z 2022-01-17T21:20:42.310968Z 2022-01-17T21:20:50.693552Z 2022-01-17T21:20:43.762952Z 2022-01-17T21:20:52.393285Z 2022-01-17T21:20:47.860020Z 2022-01-17T21:20:50.798270Z 2022-01-17T21:20:40.006616Z 2022-01-17T21:20:41.525418Z 2022-01-17T21:20:47.165539Z 2022-01-17T21:20:43.472308Z 2022-01-17T21:20:41.610055Z 2022-01-17T21:20:41.960358Z 2022-01-17T21:20:39.708282Z 2022-01-17T21:20:38.703390Z 2022-01-17T21:20:38.093964Z 2022-01-17T21:20:45.372903Z 2022-01-17T21:20:48.499231Z 2022-01-17T21:20:41.889312Z 2022-01-17T21:20:37.996870Z 2022-01-17T21:20:33.765381Z 2022-01-17T21:20:37.984502Z 2022-01-17T21:20:33.592335Z 2022-01-17T21:20:21.443902Z 2022-01-17T21:20:30.397334Z 2022-01-17T21:20:29.009909Z 2022-01-17T21:20:31.842038Z 2022-01-17T21:20:18.066043Z 2022-01-17T21:20:17.155290Z 2022-01-17T21:20:26.865942Z 2022-01-17T21:20:08.236257Z 2022-01-17T21:20:24.189968Z 2022-01-17T21:19:52.292065Z 2022-01-17T21:19:51.716625Z 2022-01-17T21:19:54.728731Z 2022-01-17T21:19:51.757003Z 2022-01-17T21:19:50.907744Z 2022-01-17T21:19:49.596324Z 2022-01-17T21:19:50.570290Z 2022-01-17T21:19:48.134272Z 2022-01-17T21:19:46.315045Z 2022-01-17T21:19:46.489393Z 2022-01-17T21:19:45.003938Z 2022-01-17T21:19:48.134272Z 2022-01-17T21:19:39.695916Z 2022-01-17T21:19:45.303230Z 2022-01-17T21:19:36.001058Z 2022-01-17T21:19:41.574761Z 2022-01-17T21:19:36.719291Z 2022-01-17T21:19:31.756736Z 2022-01-17T21:25:14.216884Z 2022-01-17T21:25:09.230424Z 2022-01-17T21:25:13.215943Z 2022-01-17T21:25:08.669358Z 2022-01-17T21:25:17.573052Z 2022-01-17T21:25:08.902269Z 2022-01-17T21:25:16.201781Z 2022-01-17T21:25:09.855470Z 2022-01-17T21:25:17.743342Z 2022-01-17T21:25:08.669358Z 2022-01-17T21:25:17.066844Z 2022-01-17T21:25:06.844302Z 2022-01-17T21:25:18.244904Z 2022-01-17T21:25:06.574298Z 2022-01-17T21:25:17.962896Z 2022-01-17T21:25:04.152528Z 2022-01-17T21:25:15.014471Z 2022-01-17T21:25:06.477190Z 2022-01-17T21:25:08.986638Z 2022-01-17T21:25:16.814774Z 2022-01-17T21:25:05.684486Z 2022-01-17T21:25:15.451180Z 2022-01-17T21:25:07.308480Z 2022-01-17T21:25:14.216884Z 2022-01-17T21:25:04.303991Z 2022-01-17T21:25:17.495052Z 2022-01-17T21:25:12.226292Z 2022-01-17T21:25:03.516995Z 2022-01-17T21:25:09.232295Z 2022-01-17T21:24:59.951920Z 2022-01-17T21:25:14.970658Z 2022-01-17T21:25:05.120697Z 2022-01-17T21:25:09.879268Z 2022-01-17T21:25:04.651419Z 2022-01-17T21:25:11.860637Z 2022-01-17T21:24:57.661221Z 2022-01-17T21:25:09.135430Z 2022-01-17T21:25:01.542198Z 2022-01-17T21:25:12.261403Z 2022-01-17T21:25:02.209912Z 2022-01-17T21:24:57.732611Z 2022-01-17T21:25:04.061875Z 2022-01-17T21:25:00.217949Z 2022-01-17T21:25:13.351334Z 2022-01-17T21:24:54.286022Z 2022-01-17T21:25:04.061875Z 2022-01-17T21:24:56.218929Z 2022-01-17T21:25:05.385283Z 2022-01-17T21:25:02.440108Z 2022-01-17T21:24:51.939983Z 2022-01-17T21:24:58.803440Z 2022-01-17T21:24:51.253958Z 2022-01-17T21:25:05.789299Z 2022-01-17T21:24:50.002352Z 2022-01-17T21:24:51.411886Z 2022-01-17T21:25:00.739419Z 2022-01-17T21:24:50.969368Z 2022-01-17T21:24:59.669699Z 2022-01-17T21:24:48.381997Z 2022-01-17T21:25:03.830720Z 2022-01-17T21:24:56.866400Z 2022-01-17T21:25:04.606038Z 2022-01-17T21:24:42.532015Z 2022-01-17T21:24:50.812312Z 2022-01-17T21:25:01.693297Z 2022-01-17T21:24:45.603899Z 2022-01-17T21:24:59.976410Z 2022-01-17T21:24:39.296908Z 2022-01-17T21:24:57.174876Z 2022-01-17T21:24:50.875292Z 2022-01-17T21:24:18.308893Z 2022-01-17T21:24:23.933311Z 2022-01-17T21:24:17.900602Z 2022-01-17T21:24:24.306439Z 2022-01-17T21:24:23.232885Z 2022-01-17T21:24:16.720100Z 2022-01-17T21:24:18.408544Z 2022-01-17T21:24:10.468900Z 2022-01-17T21:24:10.835973Z 2022-01-17T21:24:21.306522Z 2022-01-17T21:24:14.842779Z 2022-01-17T21:24:15.714298Z 2022-01-17T21:24:07.886997Z 2022-01-17T21:24:12.599349Z 2022-01-17T21:24:06.102995Z 2022-01-17T21:24:12.215308Z 2022-01-17T21:24:05.441759Z 2022-01-17T21:24:12.822062Z 2022-01-17T21:24:01.231905Z 2022-01-17T21:24:12.391850Z 2022-01-17T21:24:00.948226Z 2022-01-17T21:24:07.195728Z 2022-01-17T21:23:59.151675Z 2022-01-17T21:24:07.273259Z 2022-01-17T21:23:58.362215Z 2022-01-17T21:23:49.480325Z 2022-01-17T21:23:47.789082Z 2022-01-17T21:23:49.335678Z 2022-01-17T21:23:44.467191Z 2022-01-17T21:23:43.192361Z 2022-01-17T21:29:28.800040Z 2022-01-17T21:29:21.621177Z 2022-01-17T21:29:24.459329Z 2022-01-17T21:29:20.762423Z 2022-01-17T21:29:25.421014Z 2022-01-17T21:29:21.621177Z 2022-01-17T21:29:22.946358Z 2022-01-17T21:29:26.598719Z 2022-01-17T21:29:25.760624Z 2022-01-17T21:29:22.841504Z 2022-01-17T21:29:26.627961Z 2022-01-17T21:29:22.702087Z 2022-01-17T21:29:22.583362Z 2022-01-17T21:29:25.566529Z 2022-01-17T21:29:21.891054Z 2022-01-17T21:29:17.524494Z 2022-01-17T21:29:26.196414Z 2022-01-17T21:29:21.797281Z 2022-01-17T21:29:24.313564Z 2022-01-17T21:29:21.222998Z 2022-01-17T21:29:13.818970Z 2022-01-17T21:29:16.981320Z 2022-01-17T21:29:17.333873Z 2022-01-17T21:29:24.228305Z 2022-01-17T21:29:17.269944Z 2022-01-17T21:29:19.643950Z 2022-01-17T21:29:23.434428Z 2022-01-17T21:29:12.326952Z 2022-01-17T21:29:18.266408Z 2022-01-17T21:29:22.527288Z 2022-01-17T21:29:17.051123Z 2022-01-17T21:29:17.971291Z 2022-01-17T21:29:16.447025Z 2022-01-17T21:29:22.004114Z 2022-01-17T21:29:11.548918Z 2022-01-17T21:29:19.589303Z 2022-01-17T21:29:11.441083Z 2022-01-17T21:29:13.738479Z 2022-01-17T21:29:16.615958Z 2022-01-17T21:29:17.235582Z 2022-01-17T21:29:12.582974Z 2022-01-17T21:29:16.697560Z 2022-01-17T21:29:13.109919Z 2022-01-17T21:29:11.521264Z 2022-01-17T21:29:09.467265Z 2022-01-17T21:29:14.392473Z 2022-01-17T21:29:04.471974Z 2022-01-17T21:29:05.477341Z 2022-01-17T21:29:07.756410Z 2022-01-17T21:29:10.792677Z 2022-01-17T21:29:07.288937Z 2022-01-17T21:29:10.290414Z 2022-01-17T21:29:07.785323Z 2022-01-17T21:29:13.969579Z 2022-01-17T21:29:13.923993Z 2022-01-17T21:29:00.514654Z 2022-01-17T21:29:14.654976Z 2022-01-17T21:29:08.238492Z 2022-01-17T21:29:02.815906Z 2022-01-17T21:29:09.840180Z 2022-01-17T21:28:55.094407Z 2022-01-17T21:29:04.676061Z 2022-01-17T21:28:58.193965Z 2022-01-17T21:29:05.891323Z 2022-01-17T21:29:01.187010Z 2022-01-17T21:29:06.152137Z 2022-01-17T21:29:02.344158Z 2022-01-17T21:29:00.824868Z 2022-01-17T21:29:00.354229Z 2022-01-17T21:28:57.478329Z 2022-01-17T21:28:56.358957Z 2022-01-17T21:28:59.450283Z 2022-01-17T21:28:44.734991Z 2022-01-17T21:29:05.197689Z 2022-01-17T21:28:53.173466Z 2022-01-17T21:29:01.711333Z 2022-01-17T21:28:48.946936Z 2022-01-17T21:29:01.933830Z 2022-01-17T21:28:30.941799Z 2022-01-17T21:28:48.194337Z 2022-01-17T21:28:40.789478Z 2022-01-17T21:28:46.043332Z 2022-01-17T21:28:41.132923Z 2022-01-17T21:28:44.227399Z 2022-01-17T21:28:40.671894Z 2022-01-17T21:28:32.769334Z 2022-01-17T21:28:38.628226Z 2022-01-17T21:28:20.471489Z 2022-01-17T21:28:03.381004Z 2022-01-17T21:28:21.509284Z 2022-01-17T21:28:03.129368Z 2022-01-17T21:28:23.700330Z 2022-01-17T21:28:01.950217Z 2022-01-17T21:28:11.553175Z 2022-01-17T21:28:00.935915Z 2022-01-17T21:27:58.632461Z 2022-01-17T21:27:56.566069Z 2022-01-17T21:27:56.605681Z 2022-01-17T21:27:51.168036Z 2022-01-17T21:27:53.907322Z 2022-01-17T21:33:29.583057Z 2022-01-17T21:33:25.055020Z 2022-01-17T21:33:28.351302Z 2022-01-17T21:33:31.327051Z 2022-01-17T21:33:26.291887Z 2022-01-17T21:33:29.173831Z 2022-01-17T21:33:33.799397Z 2022-01-17T21:33:31.327051Z 2022-01-17T21:33:33.834376Z 2022-01-17T21:33:28.918064Z 2022-01-17T21:33:28.762558Z 2022-01-17T21:33:30.267940Z 2022-01-17T21:33:26.118006Z 2022-01-17T21:33:33.514291Z 2022-01-17T21:33:27.515478Z 2022-01-17T21:33:33.151405Z 2022-01-17T21:33:28.211333Z 2022-01-17T21:33:32.170676Z 2022-01-17T21:33:27.291144Z 2022-01-17T21:33:28.791603Z 2022-01-17T21:33:25.701187Z 2022-01-17T21:33:34.060942Z 2022-01-17T21:33:23.520886Z 2022-01-17T21:33:25.152953Z 2022-01-17T21:33:25.314703Z 2022-01-17T21:33:33.948002Z 2022-01-17T21:33:26.952223Z 2022-01-17T21:33:31.441930Z 2022-01-17T21:33:24.230630Z 2022-01-17T21:33:28.436595Z 2022-01-17T21:33:19.230041Z 2022-01-17T21:33:25.885952Z 2022-01-17T21:33:24.281940Z 2022-01-17T21:33:27.056323Z 2022-01-17T21:33:23.832947Z 2022-01-17T21:33:28.238873Z 2022-01-17T21:33:17.971918Z 2022-01-17T21:33:28.724114Z 2022-01-17T21:33:22.002888Z 2022-01-17T21:33:26.265379Z 2022-01-17T21:33:22.325207Z 2022-01-17T21:33:22.118253Z 2022-01-17T21:33:18.849004Z 2022-01-17T21:33:27.181402Z 2022-01-17T21:33:20.486485Z 2022-01-17T21:33:17.564261Z 2022-01-17T21:33:12.600027Z 2022-01-17T21:33:19.776312Z 2022-01-17T21:33:15.603004Z 2022-01-17T21:33:24.346416Z 2022-01-17T21:33:10.756874Z 2022-01-17T21:33:16.473315Z 2022-01-17T21:33:17.379962Z 2022-01-17T21:33:24.168137Z 2022-01-17T21:33:16.905360Z 2022-01-17T21:33:19.502287Z 2022-01-17T21:33:09.374916Z 2022-01-17T21:33:13.054587Z 2022-01-17T21:33:10.278439Z 2022-01-17T21:33:13.329306Z 2022-01-17T21:33:18.849004Z 2022-01-17T21:32:54.177657Z 2022-01-17T21:33:20.423934Z 2022-01-17T21:32:53.468974Z 2022-01-17T21:33:05.413879Z 2022-01-17T21:32:49.574704Z 2022-01-17T21:33:02.530480Z 2022-01-17T21:32:42.803471Z 2022-01-17T21:33:16.279033Z 2022-01-17T21:32:55.245471Z 2022-01-17T21:32:41.493318Z 2022-01-17T21:32:43.229593Z 2022-01-17T21:32:41.029831Z 2022-01-17T21:32:52.075560Z 2022-01-17T21:32:31.272919Z 2022-01-17T21:32:57.994278Z 2022-01-17T21:32:39.089008Z 2022-01-17T21:32:40.803319Z 2022-01-17T21:32:33.612530Z 2022-01-17T21:32:41.713373Z 2022-01-17T21:32:31.174511Z 2022-01-17T21:32:35.572860Z 2022-01-17T21:32:29.392803Z 2022-01-17T21:32:26.734583Z 2022-01-17T21:32:28.334149Z 2022-01-17T21:32:38.281258Z 2022-01-17T21:32:23.808924Z 2022-01-17T21:32:25.575458Z 2022-01-17T21:32:24.856024Z 2022-01-17T21:32:27.719865Z 2022-01-17T21:32:21.178935Z 2022-01-17T21:32:16.157298Z 2022-01-17T21:32:09.136866Z 2022-01-17T21:32:11.826449Z 2022-01-17T21:32:06.660164Z 2022-01-17T21:32:06.284069Z 2022-01-17T21:32:05.357538Z 2022-01-17T21:32:05.991330Z 2022-01-17T21:32:02.120053Z 2022-01-17T21:32:02.355454Z", + "failed_job_ids": " []", + "finished": "2022-01-17T21:15:54.652065Z 2022-01-17T21:15:25.928663Z 2022-01-17T21:15:57.392411Z 2022-01-17T21:15:10.191336Z 2022-01-17T21:15:57.571305Z 2022-01-17T21:15:18.419949Z 2022-01-17T21:15:52.222238Z 2022-01-17T21:16:02.543829Z 2022-01-17T21:15:54.571292Z 2022-01-17T21:15:54.157503Z 2022-01-17T21:15:44.674945Z 2022-01-17T21:16:47.352814Z 2022-01-17T21:15:30.240198Z 2022-01-17T21:16:29.883632Z 2022-01-17T21:15:37.428232Z 2022-01-17T21:16:38.659755Z 2022-01-17T21:15:38.945291Z 2022-01-17T21:17:16.051800Z 2022-01-17T21:15:20.972353Z 2022-01-17T21:15:43.870946Z 2022-01-17T21:15:16.273910Z 2022-01-17T21:16:56.771715Z 2022-01-17T21:15:08.105465Z 2022-01-17T21:17:36.289796Z 2022-01-17T21:14:56.391480Z 2022-01-17T21:17:26.129441Z 2022-01-17T21:15:09.751738Z 2022-01-17T21:16:12.012553Z 2022-01-17T21:14:57.575729Z 2022-01-17T21:16:21.519978Z 2022-01-17T21:14:50.558256Z 2022-01-17T21:17:06.974810Z 2022-01-17T21:14:45.746192Z 2022-01-17T21:14:54.763999Z 2022-01-17T21:14:41.906712Z 2022-01-17T21:16:22.001328Z 2022-01-17T21:15:50.111213Z 2022-01-17T21:16:21.974581Z 2022-01-17T21:16:22.043013Z 2022-01-17T21:16:21.675935Z 2022-01-17T21:15:48.693805Z 2022-01-17T21:16:19.326857Z 2022-01-17T21:15:48.085016Z 2022-01-17T21:16:20.511390Z 2022-01-17T21:15:49.275988Z 2022-01-17T21:16:19.179575Z 2022-01-17T21:15:46.845750Z 2022-01-17T21:16:22.037116Z 2022-01-17T21:15:48.243465Z 2022-01-17T21:16:19.295757Z 2022-01-17T21:15:49.237864Z 2022-01-17T21:16:20.250583Z 2022-01-17T21:15:47.058192Z 2022-01-17T21:15:47.056253Z 2022-01-17T21:16:20.339905Z 2022-01-17T21:16:20.523208Z 2022-01-17T21:16:20.921379Z 2022-01-17T21:15:46.094968Z 2022-01-17T21:16:13.327656Z 2022-01-17T21:16:11.804992Z 2022-01-17T21:16:17.693898Z 2022-01-17T21:15:41.563743Z 2022-01-17T21:16:11.323187Z 2022-01-17T21:15:37.009962Z 2022-01-17T21:16:13.074258Z 2022-01-17T21:16:15.295785Z 2022-01-17T21:15:37.848122Z 2022-01-17T21:16:12.350588Z 2022-01-17T21:15:37.164673Z 2022-01-17T21:15:36.544779Z 2022-01-17T21:15:42.137915Z 2022-01-17T21:16:14.379246Z 2022-01-17T21:15:30.127703Z 2022-01-17T21:16:15.773313Z 2022-01-17T21:16:16.068105Z 2022-01-17T21:15:37.167403Z 2022-01-17T21:16:14.588316Z 2022-01-17T21:15:26.784960Z 2022-01-17T21:16:05.408635Z 2022-01-17T21:15:32.218810Z 2022-01-17T21:16:06.828430Z 2022-01-17T21:15:32.841909Z 2022-01-17T21:16:13.321152Z 2022-01-17T21:15:37.184730Z 2022-01-17T21:16:10.036255Z 2022-01-17T21:15:35.208711Z 2022-01-17T21:16:05.851729Z 2022-01-17T21:15:29.247551Z 2022-01-17T21:16:12.002321Z 2022-01-17T21:15:28.634157Z 2022-01-17T21:16:01.787201Z 2022-01-17T21:15:29.985532Z 2022-01-17T21:15:20.061803Z 2022-01-17T21:16:00.858491Z 2022-01-17T21:16:07.780565Z 2022-01-17T21:15:18.416968Z 2022-01-17T21:16:04.632129Z 2022-01-17T21:15:16.636252Z 2022-01-17T21:15:59.032997Z 2022-01-17T21:15:24.021421Z 2022-01-17T21:21:12.291055Z 2022-01-17T21:21:13.423631Z 2022-01-17T21:21:03.959913Z 2022-01-17T21:21:06.198958Z 2022-01-17T21:21:13.117001Z 2022-01-17T21:21:05.819488Z 2022-01-17T21:21:13.220698Z 2022-01-17T21:21:05.659411Z 2022-01-17T21:21:12.941996Z 2022-01-17T21:21:04.132837Z 2022-01-17T21:21:11.681525Z 2022-01-17T21:21:05.344909Z 2022-01-17T21:21:11.480865Z 2022-01-17T21:21:03.220136Z 2022-01-17T21:21:08.902743Z 2022-01-17T21:21:02.420018Z 2022-01-17T21:21:08.693169Z 2022-01-17T21:21:09.538820Z 2022-01-17T21:21:01.719986Z 2022-01-17T21:21:12.537566Z 2022-01-17T21:21:09.351552Z 2022-01-17T21:21:03.285468Z 2022-01-17T21:21:00.864950Z 2022-01-17T21:21:06.335430Z 2022-01-17T21:20:55.464678Z 2022-01-17T21:21:06.853660Z 2022-01-17T21:21:01.538636Z 2022-01-17T21:21:03.765693Z 2022-01-17T21:21:08.415104Z 2022-01-17T21:21:11.125599Z 2022-01-17T21:21:02.190459Z 2022-01-17T21:20:54.065578Z 2022-01-17T21:21:09.545058Z 2022-01-17T21:20:58.098624Z 2022-01-17T21:21:07.291483Z 2022-01-17T21:21:01.906936Z 2022-01-17T21:20:56.627501Z 2022-01-17T21:21:01.324085Z 2022-01-17T21:21:00.633170Z 2022-01-17T21:20:56.589845Z 2022-01-17T21:20:57.211695Z 2022-01-17T21:20:56.154114Z 2022-01-17T21:21:01.483507Z 2022-01-17T21:20:55.466469Z 2022-01-17T21:21:01.212287Z 2022-01-17T21:21:02.117742Z 2022-01-17T21:21:06.819361Z 2022-01-17T21:20:54.587649Z 2022-01-17T21:21:04.533989Z 2022-01-17T21:20:53.000983Z 2022-01-17T21:21:01.130549Z 2022-01-17T21:20:48.119836Z 2022-01-17T21:20:58.254854Z 2022-01-17T21:20:48.265835Z 2022-01-17T21:20:56.642859Z 2022-01-17T21:20:53.394129Z 2022-01-17T21:20:55.149702Z 2022-01-17T21:20:44.519803Z 2022-01-17T21:20:46.999116Z 2022-01-17T21:20:53.135400Z 2022-01-17T21:20:49.810643Z 2022-01-17T21:20:46.645928Z 2022-01-17T21:20:47.126176Z 2022-01-17T21:20:46.555125Z 2022-01-17T21:20:44.779341Z 2022-01-17T21:20:42.772714Z 2022-01-17T21:20:51.356260Z 2022-01-17T21:20:53.406634Z 2022-01-17T21:20:48.202417Z 2022-01-17T21:20:44.190786Z 2022-01-17T21:20:38.858208Z 2022-01-17T21:20:42.950578Z 2022-01-17T21:20:39.100180Z 2022-01-17T21:20:27.572591Z 2022-01-17T21:20:34.838111Z 2022-01-17T21:20:32.722656Z 2022-01-17T21:20:37.717048Z 2022-01-17T21:20:24.885974Z 2022-01-17T21:20:23.406161Z 2022-01-17T21:20:31.330056Z 2022-01-17T21:20:12.507048Z 2022-01-17T21:20:29.512610Z 2022-01-17T21:19:56.924373Z 2022-01-17T21:19:55.840123Z 2022-01-17T21:19:59.503911Z 2022-01-17T21:19:55.833542Z 2022-01-17T21:19:55.257998Z 2022-01-17T21:19:53.976173Z 2022-01-17T21:19:55.124333Z 2022-01-17T21:19:52.717568Z 2022-01-17T21:19:50.169418Z 2022-01-17T21:19:50.090433Z 2022-01-17T21:19:48.802320Z 2022-01-17T21:19:52.039875Z 2022-01-17T21:19:44.117491Z 2022-01-17T21:19:49.579998Z 2022-01-17T21:19:40.397104Z 2022-01-17T21:19:45.177422Z 2022-01-17T21:19:40.773945Z 2022-01-17T21:19:36.168806Z 2022-01-17T21:25:19.625297Z 2022-01-17T21:25:13.632065Z 2022-01-17T21:25:17.903596Z 2022-01-17T21:25:12.897491Z 2022-01-17T21:25:22.651875Z 2022-01-17T21:25:13.691166Z 2022-01-17T21:25:20.337362Z 2022-01-17T21:25:13.785853Z 2022-01-17T21:25:21.881421Z 2022-01-17T21:25:12.512023Z 2022-01-17T21:25:21.850919Z 2022-01-17T21:25:11.013355Z 2022-01-17T21:25:22.684950Z 2022-01-17T21:25:10.688407Z 2022-01-17T21:25:21.898538Z 2022-01-17T21:25:09.511168Z 2022-01-17T21:25:20.162700Z 2022-01-17T21:25:11.665470Z 2022-01-17T21:25:12.880795Z 2022-01-17T21:25:21.565767Z 2022-01-17T21:25:10.595190Z 2022-01-17T21:25:20.097145Z 2022-01-17T21:25:11.412226Z 2022-01-17T21:25:18.664140Z 2022-01-17T21:25:08.275324Z 2022-01-17T21:25:21.176688Z 2022-01-17T21:25:16.363449Z 2022-01-17T21:25:08.612970Z 2022-01-17T21:25:15.084623Z 2022-01-17T21:25:04.960474Z 2022-01-17T21:25:19.713997Z 2022-01-17T21:25:09.873019Z 2022-01-17T21:25:15.072276Z 2022-01-17T21:25:09.553356Z 2022-01-17T21:25:16.628547Z 2022-01-17T21:25:03.416752Z 2022-01-17T21:25:14.328634Z 2022-01-17T21:25:06.828546Z 2022-01-17T21:25:18.077067Z 2022-01-17T21:25:06.601571Z 2022-01-17T21:25:02.426033Z 2022-01-17T21:25:07.738229Z 2022-01-17T21:25:05.730273Z 2022-01-17T21:25:17.645514Z 2022-01-17T21:24:59.226801Z 2022-01-17T21:25:10.158918Z 2022-01-17T21:25:00.658862Z 2022-01-17T21:25:09.511132Z 2022-01-17T21:25:08.442300Z 2022-01-17T21:24:57.504439Z 2022-01-17T21:25:03.965297Z 2022-01-17T21:24:57.574410Z 2022-01-17T21:25:11.319634Z 2022-01-17T21:24:54.963042Z 2022-01-17T21:24:56.974897Z 2022-01-17T21:25:07.443971Z 2022-01-17T21:24:57.421335Z 2022-01-17T21:25:07.278302Z 2022-01-17T21:24:53.620905Z 2022-01-17T21:25:07.710462Z 2022-01-17T21:25:02.668102Z 2022-01-17T21:25:10.140979Z 2022-01-17T21:24:47.682450Z 2022-01-17T21:24:56.104748Z 2022-01-17T21:25:07.289071Z 2022-01-17T21:24:50.245786Z 2022-01-17T21:25:07.260247Z 2022-01-17T21:24:43.837843Z 2022-01-17T21:25:02.247305Z 2022-01-17T21:24:56.657377Z 2022-01-17T21:24:23.677343Z 2022-01-17T21:24:30.170348Z 2022-01-17T21:24:22.560562Z 2022-01-17T21:24:30.597248Z 2022-01-17T21:24:29.390680Z 2022-01-17T21:24:20.684760Z 2022-01-17T21:24:23.619299Z 2022-01-17T21:24:15.472937Z 2022-01-17T21:24:16.078855Z 2022-01-17T21:24:27.739199Z 2022-01-17T21:24:19.996889Z 2022-01-17T21:24:22.591696Z 2022-01-17T21:24:12.506737Z 2022-01-17T21:24:18.696650Z 2022-01-17T21:24:10.609116Z 2022-01-17T21:24:18.411248Z 2022-01-17T21:24:10.028486Z 2022-01-17T21:24:18.392419Z 2022-01-17T21:24:06.221165Z 2022-01-17T21:24:16.101042Z 2022-01-17T21:24:05.839110Z 2022-01-17T21:24:12.584216Z 2022-01-17T21:24:03.410929Z 2022-01-17T21:24:13.127806Z 2022-01-17T21:24:01.963851Z 2022-01-17T21:23:53.063618Z 2022-01-17T21:23:50.255167Z 2022-01-17T21:23:53.365949Z 2022-01-17T21:23:48.172288Z 2022-01-17T21:23:47.221041Z 2022-01-17T21:29:30.644808Z 2022-01-17T21:29:25.689951Z 2022-01-17T21:29:29.278796Z 2022-01-17T21:29:25.216859Z 2022-01-17T21:29:29.512591Z 2022-01-17T21:29:25.811053Z 2022-01-17T21:29:26.953395Z 2022-01-17T21:29:30.775850Z 2022-01-17T21:29:29.928422Z 2022-01-17T21:29:26.441156Z 2022-01-17T21:29:30.812728Z 2022-01-17T21:29:26.665909Z 2022-01-17T21:29:27.755336Z 2022-01-17T21:29:30.094866Z 2022-01-17T21:29:25.996095Z 2022-01-17T21:29:23.067560Z 2022-01-17T21:29:30.141669Z 2022-01-17T21:29:26.138217Z 2022-01-17T21:29:28.960812Z 2022-01-17T21:29:26.453099Z 2022-01-17T21:29:18.707195Z 2022-01-17T21:29:22.317295Z 2022-01-17T21:29:21.976735Z 2022-01-17T21:29:29.055356Z 2022-01-17T21:29:21.704275Z 2022-01-17T21:29:24.188274Z 2022-01-17T21:29:27.980217Z 2022-01-17T21:29:17.917372Z 2022-01-17T21:29:23.461710Z 2022-01-17T21:29:27.476467Z 2022-01-17T21:29:21.652583Z 2022-01-17T21:29:23.094887Z 2022-01-17T21:29:21.702269Z 2022-01-17T21:29:26.391435Z 2022-01-17T21:29:17.749420Z 2022-01-17T21:29:24.062258Z 2022-01-17T21:29:16.349884Z 2022-01-17T21:29:20.820697Z 2022-01-17T21:29:21.866990Z 2022-01-17T21:29:21.383898Z 2022-01-17T21:29:16.883747Z 2022-01-17T21:29:22.250609Z 2022-01-17T21:29:18.012096Z 2022-01-17T21:29:16.852732Z 2022-01-17T21:29:13.764143Z 2022-01-17T21:29:18.512307Z 2022-01-17T21:29:10.171844Z 2022-01-17T21:29:12.211344Z 2022-01-17T21:29:13.565749Z 2022-01-17T21:29:18.045880Z 2022-01-17T21:29:13.275285Z 2022-01-17T21:29:16.163330Z 2022-01-17T21:29:13.512301Z 2022-01-17T21:29:21.355400Z 2022-01-17T21:29:20.697163Z 2022-01-17T21:29:06.051141Z 2022-01-17T21:29:19.195006Z 2022-01-17T21:29:14.134866Z 2022-01-17T21:29:08.463791Z 2022-01-17T21:29:16.339304Z 2022-01-17T21:29:00.955962Z 2022-01-17T21:29:10.173435Z 2022-01-17T21:29:04.810791Z 2022-01-17T21:29:13.557315Z 2022-01-17T21:29:07.234747Z 2022-01-17T21:29:11.895799Z 2022-01-17T21:29:08.765073Z 2022-01-17T21:29:07.017062Z 2022-01-17T21:29:06.782166Z 2022-01-17T21:29:03.750349Z 2022-01-17T21:29:03.296703Z 2022-01-17T21:29:06.879256Z 2022-01-17T21:28:51.637198Z 2022-01-17T21:29:10.161934Z 2022-01-17T21:28:58.986665Z 2022-01-17T21:29:07.706743Z 2022-01-17T21:28:54.393746Z 2022-01-17T21:29:07.354333Z 2022-01-17T21:28:37.030844Z 2022-01-17T21:28:52.947997Z 2022-01-17T21:28:47.165724Z 2022-01-17T21:28:51.314041Z 2022-01-17T21:28:46.640777Z 2022-01-17T21:28:50.463285Z 2022-01-17T21:28:45.910846Z 2022-01-17T21:28:39.097111Z 2022-01-17T21:28:44.481679Z 2022-01-17T21:28:26.677097Z 2022-01-17T21:28:07.782165Z 2022-01-17T21:28:27.920409Z 2022-01-17T21:28:07.182844Z 2022-01-17T21:28:29.055204Z 2022-01-17T21:28:05.625663Z 2022-01-17T21:28:16.230930Z 2022-01-17T21:28:04.760828Z 2022-01-17T21:28:02.054671Z 2022-01-17T21:28:00.411932Z 2022-01-17T21:28:00.980231Z 2022-01-17T21:27:54.993106Z 2022-01-17T21:27:58.504025Z 2022-01-17T21:33:32.890586Z 2022-01-17T21:33:29.623629Z 2022-01-17T21:33:33.927643Z 2022-01-17T21:33:35.956251Z 2022-01-17T21:33:31.089459Z 2022-01-17T21:33:33.355827Z 2022-01-17T21:33:37.511635Z 2022-01-17T21:33:36.381903Z 2022-01-17T21:33:38.083929Z 2022-01-17T21:33:33.359373Z 2022-01-17T21:33:32.609734Z 2022-01-17T21:33:34.641464Z 2022-01-17T21:33:31.095610Z 2022-01-17T21:33:37.745733Z 2022-01-17T21:33:31.638662Z 2022-01-17T21:33:37.952695Z 2022-01-17T21:33:31.587344Z 2022-01-17T21:33:36.854910Z 2022-01-17T21:33:31.959470Z 2022-01-17T21:33:33.420272Z 2022-01-17T21:33:31.019109Z 2022-01-17T21:33:38.129870Z 2022-01-17T21:33:28.024017Z 2022-01-17T21:33:29.299327Z 2022-01-17T21:33:31.038063Z 2022-01-17T21:33:38.510601Z 2022-01-17T21:33:32.225318Z 2022-01-17T21:33:35.521813Z 2022-01-17T21:33:30.074324Z 2022-01-17T21:33:32.664173Z 2022-01-17T21:33:23.289029Z 2022-01-17T21:33:31.561501Z 2022-01-17T21:33:30.483104Z 2022-01-17T21:33:32.048456Z 2022-01-17T21:33:29.285073Z 2022-01-17T21:33:33.039587Z 2022-01-17T21:33:23.659871Z 2022-01-17T21:33:33.217339Z 2022-01-17T21:33:27.826294Z 2022-01-17T21:33:32.133600Z 2022-01-17T21:33:28.202231Z 2022-01-17T21:33:26.258862Z 2022-01-17T21:33:23.325231Z 2022-01-17T21:33:32.468790Z 2022-01-17T21:33:25.308074Z 2022-01-17T21:33:22.708856Z 2022-01-17T21:33:16.356638Z 2022-01-17T21:33:24.470310Z 2022-01-17T21:33:21.683739Z 2022-01-17T21:33:29.782040Z 2022-01-17T21:33:15.601865Z 2022-01-17T21:33:23.117198Z 2022-01-17T21:33:21.518847Z 2022-01-17T21:33:29.227544Z 2022-01-17T21:33:22.487536Z 2022-01-17T21:33:25.099554Z 2022-01-17T21:33:14.736831Z 2022-01-17T21:33:19.331464Z 2022-01-17T21:33:14.869843Z 2022-01-17T21:33:18.434731Z 2022-01-17T21:33:24.142986Z 2022-01-17T21:33:00.405838Z 2022-01-17T21:33:26.780152Z 2022-01-17T21:32:58.272265Z 2022-01-17T21:33:10.382854Z 2022-01-17T21:32:54.217855Z 2022-01-17T21:33:07.536214Z 2022-01-17T21:32:47.449734Z 2022-01-17T21:33:21.612006Z 2022-01-17T21:33:01.161163Z 2022-01-17T21:32:48.252736Z 2022-01-17T21:32:50.271908Z 2022-01-17T21:32:48.223649Z 2022-01-17T21:32:57.178827Z 2022-01-17T21:32:36.965847Z 2022-01-17T21:33:02.789186Z 2022-01-17T21:32:44.660508Z 2022-01-17T21:32:48.831668Z 2022-01-17T21:32:37.933036Z 2022-01-17T21:32:48.511349Z 2022-01-17T21:32:36.625395Z 2022-01-17T21:32:40.996446Z 2022-01-17T21:32:34.276206Z 2022-01-17T21:32:31.362853Z 2022-01-17T21:32:32.596359Z 2022-01-17T21:32:45.516360Z 2022-01-17T21:32:28.109728Z 2022-01-17T21:32:30.511864Z 2022-01-17T21:32:30.331487Z 2022-01-17T21:32:33.148028Z 2022-01-17T21:32:26.352934Z 2022-01-17T21:32:20.620211Z 2022-01-17T21:32:13.133988Z 2022-01-17T21:32:16.471126Z 2022-01-17T21:32:11.416837Z 2022-01-17T21:32:10.310859Z 2022-01-17T21:32:07.261617Z 2022-01-17T21:32:10.402894Z 2022-01-17T21:32:04.239819Z 2022-01-17T21:32:04.644249Z", + "finished_diff": 111.43844439999998, + "host_status_counts": "{'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10} {'skipped': 10}", + "jobs_duration": { + "max": 186.805462, + "mean": 53.28902520000004, + "min": 10.944458 + }, + "jobs_events_duration": { + "max": 7.649114, + "mean": 3.0548032079999987, + "min": 0.166815 + }, + "jobs_events_lag": { + "max": -1.049951, + "mean": -4.967921748000001, + "min": -8.028349 + }, + "jobs_waiting": { + "max": 43.849117, + "mean": 22.692450484, + "min": 0.612525 + }, + "modified": "2022-01-17T21:14:38.389188Z 2022-01-17T21:14:38.355213Z 2022-01-17T21:14:38.321197Z 2022-01-17T21:14:38.285386Z 2022-01-17T21:14:38.253029Z 2022-01-17T21:14:38.205550Z 2022-01-17T21:14:32.416973Z 2022-01-17T21:14:32.275112Z 2022-01-17T21:14:32.123570Z 2022-01-17T21:14:31.998004Z 2022-01-17T21:14:31.876714Z 2022-01-17T21:14:31.797089Z 2022-01-17T21:14:31.737942Z 2022-01-17T21:14:31.653139Z 2022-01-17T21:14:31.556032Z 2022-01-17T21:14:31.442822Z 2022-01-17T21:14:31.345064Z 2022-01-17T21:14:31.165992Z 2022-01-17T21:14:31.060962Z 2022-01-17T21:14:30.957991Z 2022-01-17T21:14:30.866012Z 2022-01-17T21:14:30.801929Z 2022-01-17T21:14:27.977162Z 2022-01-17T21:14:27.928811Z 2022-01-17T21:14:27.768229Z 2022-01-17T21:14:27.630790Z 2022-01-17T21:14:27.485578Z 2022-01-17T21:14:27.439033Z 2022-01-17T21:14:27.373735Z 2022-01-17T21:14:27.286321Z 2022-01-17T21:14:26.370337Z 2022-01-17T21:14:26.264718Z 2022-01-17T21:14:25.704243Z 2022-01-17T21:14:25.669748Z 2022-01-17T21:14:25.189225Z 2022-01-17T21:15:18.260024Z 2022-01-17T21:15:18.205181Z 2022-01-17T21:15:18.145487Z 2022-01-17T21:15:18.049736Z 2022-01-17T21:15:17.947362Z 2022-01-17T21:15:05.665267Z 2022-01-17T21:15:05.533911Z 2022-01-17T21:15:05.378224Z 2022-01-17T21:15:05.234315Z 2022-01-17T21:15:05.122369Z 2022-01-17T21:15:04.971413Z 2022-01-17T21:15:04.824471Z 2022-01-17T21:15:04.701277Z 2022-01-17T21:15:04.598120Z 2022-01-17T21:15:04.430331Z 2022-01-17T21:15:04.224341Z 2022-01-17T21:15:04.011938Z 2022-01-17T21:15:03.426739Z 2022-01-17T21:15:03.812424Z 2022-01-17T21:15:03.676243Z 2022-01-17T21:15:03.312290Z 2022-01-17T21:15:02.194754Z 2022-01-17T21:15:03.060043Z 2022-01-17T21:15:02.710956Z 2022-01-17T21:15:02.361599Z 2022-01-17T21:15:01.942707Z 2022-01-17T21:14:39.799649Z 2022-01-17T21:14:39.760229Z 2022-01-17T21:14:39.723832Z 2022-01-17T21:14:39.534034Z 2022-01-17T21:14:39.689707Z 2022-01-17T21:14:39.653030Z 2022-01-17T21:14:39.618209Z 2022-01-17T21:14:39.581179Z 2022-01-17T21:14:39.314939Z 2022-01-17T21:14:39.489038Z 2022-01-17T21:14:39.453189Z 2022-01-17T21:14:39.405277Z 2022-01-17T21:14:39.354892Z 2022-01-17T21:14:39.277415Z 2022-01-17T21:14:39.243939Z 2022-01-17T21:14:39.211062Z 2022-01-17T21:14:39.178562Z 2022-01-17T21:14:39.146541Z 2022-01-17T21:14:39.114013Z 2022-01-17T21:14:39.081739Z 2022-01-17T21:14:39.047893Z 2022-01-17T21:14:39.012330Z 2022-01-17T21:14:38.970264Z 2022-01-17T21:14:38.935772Z 2022-01-17T21:14:38.900724Z 2022-01-17T21:14:38.863037Z 2022-01-17T21:14:38.826945Z 2022-01-17T21:14:38.791748Z 2022-01-17T21:14:38.755938Z 2022-01-17T21:14:38.721725Z 2022-01-17T21:14:38.687935Z 2022-01-17T21:14:38.624359Z 2022-01-17T21:14:38.656004Z 2022-01-17T21:14:38.593439Z 2022-01-17T21:14:38.562656Z 2022-01-17T21:14:38.527971Z 2022-01-17T21:14:38.492371Z 2022-01-17T21:14:38.459418Z 2022-01-17T21:14:38.422519Z 2022-01-17T21:20:15.554774Z 2022-01-17T21:20:15.241744Z 2022-01-17T21:20:15.163185Z 2022-01-17T21:20:15.343454Z 2022-01-17T21:20:15.089166Z 2022-01-17T21:20:15.011278Z 2022-01-17T21:20:14.925401Z 2022-01-17T21:20:14.830290Z 2022-01-17T21:20:14.745305Z 2022-01-17T21:20:14.661350Z 2022-01-17T21:20:14.587459Z 2022-01-17T21:20:14.471326Z 2022-01-17T21:20:14.339236Z 2022-01-17T21:19:45.461998Z 2022-01-17T21:19:45.407023Z 2022-01-17T21:19:45.359118Z 2022-01-17T21:19:45.279701Z 2022-01-17T21:19:44.980147Z 2022-01-17T21:19:45.235874Z 2022-01-17T21:20:14.229569Z 2022-01-17T21:19:45.172953Z 2022-01-17T21:19:45.100544Z 2022-01-17T21:19:44.878021Z 2022-01-17T21:19:44.796494Z 2022-01-17T21:19:44.655899Z 2022-01-17T21:19:44.539452Z 2022-01-17T21:19:44.430051Z 2022-01-17T21:19:44.224947Z 2022-01-17T21:19:44.352108Z 2022-01-17T21:20:14.126951Z 2022-01-17T21:19:44.294740Z 2022-01-17T21:19:44.184363Z 2022-01-17T21:19:44.128424Z 2022-01-17T21:19:44.013299Z 2022-01-17T21:19:43.902847Z 2022-01-17T21:19:43.792910Z 2022-01-17T21:19:43.701034Z 2022-01-17T21:19:43.644793Z 2022-01-17T21:19:43.587738Z 2022-01-17T21:19:43.532204Z 2022-01-17T21:19:43.476736Z 2022-01-17T21:19:43.423823Z 2022-01-17T21:19:43.368736Z 2022-01-17T21:19:43.332066Z 2022-01-17T21:19:43.300282Z 2022-01-17T21:19:43.235900Z 2022-01-17T21:19:43.145893Z 2022-01-17T21:19:43.056962Z 2022-01-17T21:19:42.967962Z 2022-01-17T21:19:42.885163Z 2022-01-17T21:19:42.852623Z 2022-01-17T21:19:42.810158Z 2022-01-17T21:19:42.769877Z 2022-01-17T21:19:42.735839Z 2022-01-17T21:19:42.699056Z 2022-01-17T21:19:42.635493Z 2022-01-17T21:19:42.601031Z 2022-01-17T21:19:42.555733Z 2022-01-17T21:19:42.507816Z 2022-01-17T21:19:42.468800Z 2022-01-17T21:19:42.432170Z 2022-01-17T21:19:42.391358Z 2022-01-17T21:19:42.336544Z 2022-01-17T21:19:42.180164Z 2022-01-17T21:19:42.137849Z 2022-01-17T21:19:42.297395Z 2022-01-17T21:19:42.263281Z 2022-01-17T21:19:42.220960Z 2022-01-17T21:19:42.008872Z 2022-01-17T21:19:41.862840Z 2022-01-17T21:19:41.752939Z 2022-01-17T21:19:41.657263Z 2022-01-17T21:19:41.520636Z 2022-01-17T21:19:41.417099Z 2022-01-17T21:19:41.237896Z 2022-01-17T21:19:41.099583Z 2022-01-17T21:19:40.891291Z 2022-01-17T21:19:40.689223Z 2022-01-17T21:19:40.537369Z 2022-01-17T21:19:40.412919Z 2022-01-17T21:19:40.271073Z 2022-01-17T21:19:40.126011Z 2022-01-17T21:19:29.863725Z 2022-01-17T21:19:29.718983Z 2022-01-17T21:19:29.544673Z 2022-01-17T21:19:29.419899Z 2022-01-17T21:19:29.181054Z 2022-01-17T21:19:28.986101Z 2022-01-17T21:19:28.808622Z 2022-01-17T21:19:28.552507Z 2022-01-17T21:19:28.386907Z 2022-01-17T21:19:28.326188Z 2022-01-17T21:19:28.137817Z 2022-01-17T21:19:27.978369Z 2022-01-17T21:19:26.046902Z 2022-01-17T21:19:27.866737Z 2022-01-17T21:19:25.982071Z 2022-01-17T21:19:25.914091Z 2022-01-17T21:19:24.858584Z 2022-01-17T21:19:24.890662Z 2022-01-17T21:24:00.145355Z 2022-01-17T21:24:00.050577Z 2022-01-17T21:23:59.955483Z 2022-01-17T21:23:59.868304Z 2022-01-17T21:23:59.780404Z 2022-01-17T21:23:59.697369Z 2022-01-17T21:23:59.618284Z 2022-01-17T21:23:59.536263Z 2022-01-17T21:23:59.451463Z 2022-01-17T21:23:59.368230Z 2022-01-17T21:23:59.275507Z 2022-01-17T21:23:59.189296Z 2022-01-17T21:23:59.072436Z 2022-01-17T21:23:58.964469Z 2022-01-17T21:23:58.864370Z 2022-01-17T21:23:58.784363Z 2022-01-17T21:23:58.706174Z 2022-01-17T21:23:57.942382Z 2022-01-17T21:23:58.629515Z 2022-01-17T21:23:58.557236Z 2022-01-17T21:23:58.480517Z 2022-01-17T21:23:58.404301Z 2022-01-17T21:23:58.317519Z 2022-01-17T21:23:58.234438Z 2022-01-17T21:23:58.147234Z 2022-01-17T21:23:58.054373Z 2022-01-17T21:23:57.854391Z 2022-01-17T21:23:57.775145Z 2022-01-17T21:23:57.689440Z 2022-01-17T21:23:57.603233Z 2022-01-17T21:23:57.527273Z 2022-01-17T21:23:57.442360Z 2022-01-17T21:23:57.353294Z 2022-01-17T21:23:57.259370Z 2022-01-17T21:23:57.176203Z 2022-01-17T21:23:57.098303Z 2022-01-17T21:23:57.013193Z 2022-01-17T21:23:56.911553Z 2022-01-17T21:23:56.816338Z 2022-01-17T21:23:56.721419Z 2022-01-17T21:23:56.204802Z 2022-01-17T21:23:56.628789Z 2022-01-17T21:23:56.528184Z 2022-01-17T21:23:56.478279Z 2022-01-17T21:23:56.425342Z 2022-01-17T21:23:56.359365Z 2022-01-17T21:23:56.299294Z 2022-01-17T21:23:56.243385Z 2022-01-17T21:23:56.154301Z 2022-01-17T21:23:56.107112Z 2022-01-17T21:23:56.068625Z 2022-01-17T21:23:56.032124Z 2022-01-17T21:23:55.992249Z 2022-01-17T21:23:55.292979Z 2022-01-17T21:23:55.947457Z 2022-01-17T21:23:55.904161Z 2022-01-17T21:23:55.860331Z 2022-01-17T21:23:55.818421Z 2022-01-17T21:23:55.756842Z 2022-01-17T21:23:55.691864Z 2022-01-17T21:23:55.623539Z 2022-01-17T21:23:55.578460Z 2022-01-17T21:23:55.465572Z 2022-01-17T21:23:55.231724Z 2022-01-17T21:23:55.178126Z 2022-01-17T21:23:55.106941Z 2022-01-17T21:23:55.043301Z 2022-01-17T21:23:54.995268Z 2022-01-17T21:23:54.942373Z 2022-01-17T21:23:54.877451Z 2022-01-17T21:23:45.291364Z 2022-01-17T21:23:45.181739Z 2022-01-17T21:23:44.910061Z 2022-01-17T21:23:44.733156Z 2022-01-17T21:23:44.328656Z 2022-01-17T21:23:44.177762Z 2022-01-17T21:23:44.013450Z 2022-01-17T21:23:44.118307Z 2022-01-17T21:23:43.918750Z 2022-01-17T21:23:43.660069Z 2022-01-17T21:23:43.443648Z 2022-01-17T21:23:43.167105Z 2022-01-17T21:23:42.673056Z 2022-01-17T21:23:42.141526Z 2022-01-17T21:23:41.737070Z 2022-01-17T21:23:41.388244Z 2022-01-17T21:23:41.186882Z 2022-01-17T21:23:41.100625Z 2022-01-17T21:23:41.042380Z 2022-01-17T21:23:40.959452Z 2022-01-17T21:23:40.886389Z 2022-01-17T21:23:40.650877Z 2022-01-17T21:23:40.429959Z 2022-01-17T21:23:40.312304Z 2022-01-17T21:23:40.094323Z 2022-01-17T21:23:35.058485Z 2022-01-17T21:23:34.897053Z 2022-01-17T21:23:34.797443Z 2022-01-17T21:23:34.664401Z 2022-01-17T21:23:33.777203Z 2022-01-17T21:28:30.435737Z 2022-01-17T21:28:30.284387Z 2022-01-17T21:28:30.179915Z 2022-01-17T21:28:30.101662Z 2022-01-17T21:28:30.004417Z 2022-01-17T21:28:29.907326Z 2022-01-17T21:28:29.746309Z 2022-01-17T21:28:29.826445Z 2022-01-17T21:28:29.628435Z 2022-01-17T21:28:29.483382Z 2022-01-17T21:28:29.409573Z 2022-01-17T21:28:29.332415Z 2022-01-17T21:28:28.769347Z 2022-01-17T21:28:29.256349Z 2022-01-17T21:28:29.161383Z 2022-01-17T21:27:58.008399Z 2022-01-17T21:28:29.018418Z 2022-01-17T21:28:28.893934Z 2022-01-17T21:27:57.972853Z 2022-01-17T21:28:28.649707Z 2022-01-17T21:27:57.936596Z 2022-01-17T21:27:57.904077Z 2022-01-17T21:27:57.868415Z 2022-01-17T21:27:57.833512Z 2022-01-17T21:27:57.598270Z 2022-01-17T21:27:57.793292Z 2022-01-17T21:27:57.703676Z 2022-01-17T21:27:57.665799Z 2022-01-17T21:27:57.630863Z 2022-01-17T21:27:57.561993Z 2022-01-17T21:27:57.526960Z 2022-01-17T21:27:57.490555Z 2022-01-17T21:27:57.450433Z 2022-01-17T21:27:57.402852Z 2022-01-17T21:27:57.328045Z 2022-01-17T21:27:57.271015Z 2022-01-17T21:27:57.222493Z 2022-01-17T21:27:57.185580Z 2022-01-17T21:27:57.150647Z 2022-01-17T21:27:57.116529Z 2022-01-17T21:27:57.082780Z 2022-01-17T21:27:57.046938Z 2022-01-17T21:27:57.014497Z 2022-01-17T21:27:56.978991Z 2022-01-17T21:27:56.943015Z 2022-01-17T21:27:56.886633Z 2022-01-17T21:27:56.836604Z 2022-01-17T21:27:56.781994Z 2022-01-17T21:27:56.736954Z 2022-01-17T21:27:56.698650Z 2022-01-17T21:27:56.636861Z 2022-01-17T21:27:56.544006Z 2022-01-17T21:27:56.490364Z 2022-01-17T21:27:56.449714Z 2022-01-17T21:27:56.352522Z 2022-01-17T21:27:56.398346Z 2022-01-17T21:27:56.313833Z 2022-01-17T21:27:56.278757Z 2022-01-17T21:27:56.241917Z 2022-01-17T21:27:56.206805Z 2022-01-17T21:27:56.169867Z 2022-01-17T21:27:56.099930Z 2022-01-17T21:27:56.048842Z 2022-01-17T21:27:55.984838Z 2022-01-17T21:27:55.909547Z 2022-01-17T21:27:55.822926Z 2022-01-17T21:27:55.759128Z 2022-01-17T21:27:55.619965Z 2022-01-17T21:27:55.693026Z 2022-01-17T21:27:55.542104Z 2022-01-17T21:27:55.480425Z 2022-01-17T21:27:55.423000Z 2022-01-17T21:27:55.326714Z 2022-01-17T21:27:55.273478Z 2022-01-17T21:27:55.219164Z 2022-01-17T21:27:55.158533Z 2022-01-17T21:27:55.098808Z 2022-01-17T21:27:55.043849Z 2022-01-17T21:27:54.976225Z 2022-01-17T21:27:54.912137Z 2022-01-17T21:27:54.847100Z 2022-01-17T21:27:54.790008Z 2022-01-17T21:27:54.729319Z 2022-01-17T21:27:54.635215Z 2022-01-17T21:27:54.564721Z 2022-01-17T21:27:54.518988Z 2022-01-17T21:27:54.474276Z 2022-01-17T21:27:47.904888Z 2022-01-17T21:27:47.750410Z 2022-01-17T21:27:47.344302Z 2022-01-17T21:27:46.877328Z 2022-01-17T21:27:46.664050Z 2022-01-17T21:27:46.458971Z 2022-01-17T21:27:46.106259Z 2022-01-17T21:27:45.854236Z 2022-01-17T21:27:43.641862Z 2022-01-17T21:27:43.608390Z 2022-01-17T21:27:42.951416Z 2022-01-17T21:27:42.918175Z 2022-01-17T21:27:42.887478Z 2022-01-17T21:32:19.288186Z 2022-01-17T21:32:19.114335Z 2022-01-17T21:32:19.202247Z 2022-01-17T21:32:19.026266Z 2022-01-17T21:32:18.949224Z 2022-01-17T21:32:18.767472Z 2022-01-17T21:32:18.868207Z 2022-01-17T21:32:18.473290Z 2022-01-17T21:32:18.657267Z 2022-01-17T21:32:18.553258Z 2022-01-17T21:32:18.387239Z 2022-01-17T21:32:18.300410Z 2022-01-17T21:32:18.213336Z 2022-01-17T21:32:18.119572Z 2022-01-17T21:32:18.035271Z 2022-01-17T21:32:17.959957Z 2022-01-17T21:32:17.903302Z 2022-01-17T21:32:17.839654Z 2022-01-17T21:32:17.785434Z 2022-01-17T21:32:17.746402Z 2022-01-17T21:32:17.697326Z 2022-01-17T21:32:17.642448Z 2022-01-17T21:32:17.582300Z 2022-01-17T21:32:17.471297Z 2022-01-17T21:32:17.414302Z 2022-01-17T21:32:17.525402Z 2022-01-17T21:32:17.358272Z 2022-01-17T21:32:17.311363Z 2022-01-17T21:32:17.219231Z 2022-01-17T21:32:17.126230Z 2022-01-17T21:32:17.028241Z 2022-01-17T21:32:16.964545Z 2022-01-17T21:32:16.879353Z 2022-01-17T21:32:16.773638Z 2022-01-17T21:32:16.682242Z 2022-01-17T21:32:16.593213Z 2022-01-17T21:32:16.468340Z 2022-01-17T21:32:16.358377Z 2022-01-17T21:32:16.307241Z 2022-01-17T21:32:16.263280Z 2022-01-17T21:32:16.140592Z 2022-01-17T21:32:16.220375Z 2022-01-17T21:32:16.006801Z 2022-01-17T21:32:15.851388Z 2022-01-17T21:32:15.704479Z 2022-01-17T21:32:15.625416Z 2022-01-17T21:32:15.527420Z 2022-01-17T21:32:15.351351Z 2022-01-17T21:32:15.237489Z 2022-01-17T21:32:15.101376Z 2022-01-17T21:32:14.954292Z 2022-01-17T21:32:14.849500Z 2022-01-17T21:32:14.701288Z 2022-01-17T21:32:14.598852Z 2022-01-17T21:32:14.507700Z 2022-01-17T21:32:14.381865Z 2022-01-17T21:32:14.279348Z 2022-01-17T21:32:14.205370Z 2022-01-17T21:32:14.132480Z 2022-01-17T21:32:14.058259Z 2022-01-17T21:32:13.976224Z 2022-01-17T21:32:03.650331Z 2022-01-17T21:32:13.888230Z 2022-01-17T21:32:03.597922Z 2022-01-17T21:32:13.771361Z 2022-01-17T21:32:03.491523Z 2022-01-17T21:32:03.431380Z 2022-01-17T21:32:03.378052Z 2022-01-17T21:32:13.651489Z 2022-01-17T21:32:03.189713Z 2022-01-17T21:32:03.133125Z 2022-01-17T21:32:03.092554Z 2022-01-17T21:32:03.041391Z 2022-01-17T21:32:02.921054Z 2022-01-17T21:32:02.778196Z 2022-01-17T21:32:02.707854Z 2022-01-17T21:32:02.610839Z 2022-01-17T21:32:02.413113Z 2022-01-17T21:32:02.251188Z 2022-01-17T21:32:02.153479Z 2022-01-17T21:32:01.970969Z 2022-01-17T21:32:01.800676Z 2022-01-17T21:32:01.714921Z 2022-01-17T21:32:01.612788Z 2022-01-17T21:32:01.391546Z 2022-01-17T21:32:01.213062Z 2022-01-17T21:32:01.065121Z 2022-01-17T21:32:00.891000Z 2022-01-17T21:32:00.227967Z 2022-01-17T21:32:00.074668Z 2022-01-17T21:31:59.885463Z 2022-01-17T21:31:53.335297Z 2022-01-17T21:31:53.085022Z 2022-01-17T21:31:52.971864Z 2022-01-17T21:31:52.775939Z 2022-01-17T21:31:51.619771Z 2022-01-17T21:31:51.586955Z 2022-01-17T21:31:51.551257Z 2022-01-17T21:31:50.965501Z 2022-01-17T21:31:50.926085Z", + "modified_diff": 41.2091002, + "started": "2022-01-17T21:14:41.371217Z 2022-01-17T21:14:40.651555Z 2022-01-17T21:14:41.131652Z 2022-01-17T21:14:40.400756Z 2022-01-17T21:14:41.374951Z 2022-01-17T21:14:40.160637Z 2022-01-17T21:14:37.995024Z 2022-01-17T21:14:36.770621Z 2022-01-17T21:14:37.168449Z 2022-01-17T21:14:36.541414Z 2022-01-17T21:14:36.019683Z 2022-01-17T21:14:35.600896Z 2022-01-17T21:14:35.590863Z 2022-01-17T21:14:35.265249Z 2022-01-17T21:14:34.592056Z 2022-01-17T21:14:34.290861Z 2022-01-17T21:14:34.445036Z 2022-01-17T21:14:33.755930Z 2022-01-17T21:14:33.546131Z 2022-01-17T21:14:33.222478Z 2022-01-17T21:14:33.192144Z 2022-01-17T21:14:32.948832Z 2022-01-17T21:14:29.878977Z 2022-01-17T21:14:29.484334Z 2022-01-17T21:14:29.267052Z 2022-01-17T21:14:29.213799Z 2022-01-17T21:14:28.984171Z 2022-01-17T21:14:28.616653Z 2022-01-17T21:14:28.531257Z 2022-01-17T21:14:28.236068Z 2022-01-17T21:14:26.887190Z 2022-01-17T21:14:26.786293Z 2022-01-17T21:14:26.004140Z 2022-01-17T21:14:25.888219Z 2022-01-17T21:14:25.383438Z 2022-01-17T21:15:19.510108Z 2022-01-17T21:15:19.191205Z 2022-01-17T21:15:19.121156Z 2022-01-17T21:15:18.883301Z 2022-01-17T21:15:18.638179Z 2022-01-17T21:15:14.065934Z 2022-01-17T21:15:13.917286Z 2022-01-17T21:15:13.116762Z 2022-01-17T21:15:12.756976Z 2022-01-17T21:15:12.296739Z 2022-01-17T21:15:12.011890Z 2022-01-17T21:15:11.276733Z 2022-01-17T21:15:10.654176Z 2022-01-17T21:15:10.366797Z 2022-01-17T21:15:10.136962Z 2022-01-17T21:15:09.221803Z 2022-01-17T21:15:09.479191Z 2022-01-17T21:15:08.055423Z 2022-01-17T21:15:08.743901Z 2022-01-17T21:15:08.413573Z 2022-01-17T21:15:08.017358Z 2022-01-17T21:15:06.707202Z 2022-01-17T21:15:07.617037Z 2022-01-17T21:15:07.211162Z 2022-01-17T21:15:07.017878Z 2022-01-17T21:15:06.443541Z 2022-01-17T21:14:52.125394Z 2022-01-17T21:14:56.522999Z 2022-01-17T21:14:51.830938Z 2022-01-17T21:14:55.659279Z 2022-01-17T21:14:55.985788Z 2022-01-17T21:14:51.270817Z 2022-01-17T21:14:55.822605Z 2022-01-17T21:14:50.788561Z 2022-01-17T21:14:48.169524Z 2022-01-17T21:14:50.046897Z 2022-01-17T21:14:54.991319Z 2022-01-17T21:14:48.895878Z 2022-01-17T21:14:53.676183Z 2022-01-17T21:14:51.641138Z 2022-01-17T21:14:47.810905Z 2022-01-17T21:14:50.955646Z 2022-01-17T21:14:47.187973Z 2022-01-17T21:14:48.901014Z 2022-01-17T21:14:46.986333Z 2022-01-17T21:14:49.651292Z 2022-01-17T21:14:46.539908Z 2022-01-17T21:14:48.738100Z 2022-01-17T21:14:46.151734Z 2022-01-17T21:14:47.803635Z 2022-01-17T21:14:45.565832Z 2022-01-17T21:14:47.026718Z 2022-01-17T21:14:44.903934Z 2022-01-17T21:14:46.557039Z 2022-01-17T21:14:44.278882Z 2022-01-17T21:14:43.606333Z 2022-01-17T21:14:42.059709Z 2022-01-17T21:14:41.759961Z 2022-01-17T21:14:43.515475Z 2022-01-17T21:14:43.294260Z 2022-01-17T21:14:41.289060Z 2022-01-17T21:14:42.486453Z 2022-01-17T21:14:40.999151Z 2022-01-17T21:14:41.799510Z 2022-01-17T21:14:40.847069Z 2022-01-17T21:20:21.165918Z 2022-01-17T21:20:20.730732Z 2022-01-17T21:20:20.060823Z 2022-01-17T21:20:21.013441Z 2022-01-17T21:20:19.904379Z 2022-01-17T21:20:19.196719Z 2022-01-17T21:20:18.911278Z 2022-01-17T21:20:18.447759Z 2022-01-17T21:20:18.261312Z 2022-01-17T21:20:17.491655Z 2022-01-17T21:20:17.244904Z 2022-01-17T21:20:16.703630Z 2022-01-17T21:20:16.611124Z 2022-01-17T21:20:06.838481Z 2022-01-17T21:20:06.309908Z 2022-01-17T21:20:05.847572Z 2022-01-17T21:20:05.766131Z 2022-01-17T21:20:04.587265Z 2022-01-17T21:20:05.416423Z 2022-01-17T21:20:16.412225Z 2022-01-17T21:20:05.132817Z 2022-01-17T21:20:05.147900Z 2022-01-17T21:20:04.331039Z 2022-01-17T21:20:03.935584Z 2022-01-17T21:20:03.730623Z 2022-01-17T21:20:03.567376Z 2022-01-17T21:20:03.308894Z 2022-01-17T21:20:02.299151Z 2022-01-17T21:20:02.835685Z 2022-01-17T21:20:15.986320Z 2022-01-17T21:20:02.984076Z 2022-01-17T21:20:02.075839Z 2022-01-17T21:20:01.782765Z 2022-01-17T21:20:01.260133Z 2022-01-17T21:20:01.247510Z 2022-01-17T21:20:00.568775Z 2022-01-17T21:20:00.445666Z 2022-01-17T21:19:59.973755Z 2022-01-17T21:19:59.853286Z 2022-01-17T21:19:59.358841Z 2022-01-17T21:19:59.497278Z 2022-01-17T21:19:58.952732Z 2022-01-17T21:19:58.532638Z 2022-01-17T21:19:58.206740Z 2022-01-17T21:19:57.828637Z 2022-01-17T21:19:57.349385Z 2022-01-17T21:19:57.217487Z 2022-01-17T21:19:56.704787Z 2022-01-17T21:19:56.528484Z 2022-01-17T21:19:56.044252Z 2022-01-17T21:19:55.330690Z 2022-01-17T21:19:55.165224Z 2022-01-17T21:19:55.077335Z 2022-01-17T21:19:54.846104Z 2022-01-17T21:19:54.159889Z 2022-01-17T21:19:54.051107Z 2022-01-17T21:19:53.492265Z 2022-01-17T21:19:53.366822Z 2022-01-17T21:19:53.102064Z 2022-01-17T21:19:52.656838Z 2022-01-17T21:19:52.393968Z 2022-01-17T21:19:52.149744Z 2022-01-17T21:19:51.856516Z 2022-01-17T21:19:50.116673Z 2022-01-17T21:19:49.798181Z 2022-01-17T21:19:51.413027Z 2022-01-17T21:19:51.398891Z 2022-01-17T21:19:51.101230Z 2022-01-17T21:19:49.289003Z 2022-01-17T21:19:49.331427Z 2022-01-17T21:19:48.807146Z 2022-01-17T21:19:48.369840Z 2022-01-17T21:19:48.251262Z 2022-01-17T21:19:47.941417Z 2022-01-17T21:19:47.685213Z 2022-01-17T21:19:47.462564Z 2022-01-17T21:19:47.341959Z 2022-01-17T21:19:47.033877Z 2022-01-17T21:19:46.640554Z 2022-01-17T21:19:46.476800Z 2022-01-17T21:19:46.232916Z 2022-01-17T21:19:46.000198Z 2022-01-17T21:19:37.537816Z 2022-01-17T21:19:37.209182Z 2022-01-17T21:19:36.722322Z 2022-01-17T21:19:36.096170Z 2022-01-17T21:19:35.277985Z 2022-01-17T21:19:34.190115Z 2022-01-17T21:19:33.623633Z 2022-01-17T21:19:32.937635Z 2022-01-17T21:19:32.037127Z 2022-01-17T21:19:31.543165Z 2022-01-17T21:19:31.390616Z 2022-01-17T21:19:31.192703Z 2022-01-17T21:19:26.971721Z 2022-01-17T21:19:30.662276Z 2022-01-17T21:19:26.707614Z 2022-01-17T21:19:26.670645Z 2022-01-17T21:19:25.164672Z 2022-01-17T21:19:25.224348Z 2022-01-17T21:24:28.176039Z 2022-01-17T21:24:27.284662Z 2022-01-17T21:24:26.990591Z 2022-01-17T21:24:26.403830Z 2022-01-17T21:24:25.978051Z 2022-01-17T21:24:25.367525Z 2022-01-17T21:24:25.641421Z 2022-01-17T21:24:24.860345Z 2022-01-17T21:24:24.265227Z 2022-01-17T21:24:23.471651Z 2022-01-17T21:24:23.508220Z 2022-01-17T21:24:23.185654Z 2022-01-17T21:24:23.425026Z 2022-01-17T21:24:22.465963Z 2022-01-17T21:24:22.156807Z 2022-01-17T21:24:21.398506Z 2022-01-17T21:24:21.035200Z 2022-01-17T21:24:17.633296Z 2022-01-17T21:24:20.510766Z 2022-01-17T21:24:20.312907Z 2022-01-17T21:24:19.910102Z 2022-01-17T21:24:20.364392Z 2022-01-17T21:24:19.269899Z 2022-01-17T21:24:19.320128Z 2022-01-17T21:24:18.458948Z 2022-01-17T21:24:18.601537Z 2022-01-17T21:24:17.608014Z 2022-01-17T21:24:16.349951Z 2022-01-17T21:24:15.718205Z 2022-01-17T21:24:15.607957Z 2022-01-17T21:24:15.279742Z 2022-01-17T21:24:14.525537Z 2022-01-17T21:24:14.482251Z 2022-01-17T21:24:13.287372Z 2022-01-17T21:24:12.774740Z 2022-01-17T21:24:11.774156Z 2022-01-17T21:24:11.772714Z 2022-01-17T21:24:11.225493Z 2022-01-17T21:24:11.362566Z 2022-01-17T21:24:10.964341Z 2022-01-17T21:24:08.001875Z 2022-01-17T21:24:10.668160Z 2022-01-17T21:24:10.506963Z 2022-01-17T21:24:10.302032Z 2022-01-17T21:24:09.860847Z 2022-01-17T21:24:09.521304Z 2022-01-17T21:24:08.898673Z 2022-01-17T21:24:08.767538Z 2022-01-17T21:24:07.641638Z 2022-01-17T21:24:06.773382Z 2022-01-17T21:24:06.413743Z 2022-01-17T21:24:05.685920Z 2022-01-17T21:24:05.504373Z 2022-01-17T21:24:02.238252Z 2022-01-17T21:24:04.583656Z 2022-01-17T21:24:04.319623Z 2022-01-17T21:24:03.913057Z 2022-01-17T21:24:03.650162Z 2022-01-17T21:24:03.314047Z 2022-01-17T21:24:03.185511Z 2022-01-17T21:24:03.045870Z 2022-01-17T21:24:02.724212Z 2022-01-17T21:24:02.461830Z 2022-01-17T21:24:01.944129Z 2022-01-17T21:24:01.898149Z 2022-01-17T21:24:01.464035Z 2022-01-17T21:24:01.302918Z 2022-01-17T21:24:01.039680Z 2022-01-17T21:24:00.816956Z 2022-01-17T21:24:00.511540Z 2022-01-17T21:23:51.365775Z 2022-01-17T21:23:51.019696Z 2022-01-17T21:23:50.755861Z 2022-01-17T21:23:50.559119Z 2022-01-17T21:23:50.289081Z 2022-01-17T21:23:50.126888Z 2022-01-17T21:23:49.229731Z 2022-01-17T21:23:49.551811Z 2022-01-17T21:23:48.864640Z 2022-01-17T21:23:48.778259Z 2022-01-17T21:23:48.521513Z 2022-01-17T21:23:48.234024Z 2022-01-17T21:23:48.074632Z 2022-01-17T21:23:48.024637Z 2022-01-17T21:23:47.538367Z 2022-01-17T21:23:47.290189Z 2022-01-17T21:23:47.047844Z 2022-01-17T21:23:47.035661Z 2022-01-17T21:23:46.531833Z 2022-01-17T21:23:46.481651Z 2022-01-17T21:23:46.295953Z 2022-01-17T21:23:46.003764Z 2022-01-17T21:23:45.888838Z 2022-01-17T21:23:45.802794Z 2022-01-17T21:23:45.658604Z 2022-01-17T21:23:36.912751Z 2022-01-17T21:23:36.488477Z 2022-01-17T21:23:36.165672Z 2022-01-17T21:23:35.549539Z 2022-01-17T21:23:33.995293Z 2022-01-17T21:28:36.377424Z 2022-01-17T21:28:36.156320Z 2022-01-17T21:28:35.844773Z 2022-01-17T21:28:35.267080Z 2022-01-17T21:28:35.100186Z 2022-01-17T21:28:34.803100Z 2022-01-17T21:28:34.083657Z 2022-01-17T21:28:34.370270Z 2022-01-17T21:28:33.750069Z 2022-01-17T21:28:33.169049Z 2022-01-17T21:28:32.781227Z 2022-01-17T21:28:32.465716Z 2022-01-17T21:28:31.060307Z 2022-01-17T21:28:32.109414Z 2022-01-17T21:28:31.878674Z 2022-01-17T21:28:19.994415Z 2022-01-17T21:28:31.662982Z 2022-01-17T21:28:31.360903Z 2022-01-17T21:28:19.562981Z 2022-01-17T21:28:30.943629Z 2022-01-17T21:28:19.408846Z 2022-01-17T21:28:18.419719Z 2022-01-17T21:28:18.404773Z 2022-01-17T21:28:17.420162Z 2022-01-17T21:28:15.399048Z 2022-01-17T21:28:17.118816Z 2022-01-17T21:28:16.565377Z 2022-01-17T21:28:16.209463Z 2022-01-17T21:28:15.832626Z 2022-01-17T21:28:15.350147Z 2022-01-17T21:28:14.921500Z 2022-01-17T21:28:14.634238Z 2022-01-17T21:28:14.657940Z 2022-01-17T21:28:14.277986Z 2022-01-17T21:28:13.930140Z 2022-01-17T21:28:13.690160Z 2022-01-17T21:28:13.791388Z 2022-01-17T21:28:13.265135Z 2022-01-17T21:28:12.982222Z 2022-01-17T21:28:12.506417Z 2022-01-17T21:28:12.299239Z 2022-01-17T21:28:11.800561Z 2022-01-17T21:28:11.582786Z 2022-01-17T21:28:11.430223Z 2022-01-17T21:28:11.297285Z 2022-01-17T21:28:10.777977Z 2022-01-17T21:28:10.404488Z 2022-01-17T21:28:09.882233Z 2022-01-17T21:28:09.640873Z 2022-01-17T21:28:09.392243Z 2022-01-17T21:28:08.880182Z 2022-01-17T21:28:08.714327Z 2022-01-17T21:28:08.640279Z 2022-01-17T21:28:08.004220Z 2022-01-17T21:28:07.846979Z 2022-01-17T21:28:07.861470Z 2022-01-17T21:28:07.419942Z 2022-01-17T21:28:07.126403Z 2022-01-17T21:28:06.751878Z 2022-01-17T21:28:06.508902Z 2022-01-17T21:28:06.194924Z 2022-01-17T21:28:05.927211Z 2022-01-17T21:28:05.538930Z 2022-01-17T21:28:05.349711Z 2022-01-17T21:28:05.050629Z 2022-01-17T21:28:04.576139Z 2022-01-17T21:28:04.426648Z 2022-01-17T21:28:04.000348Z 2022-01-17T21:28:04.043092Z 2022-01-17T21:28:03.336299Z 2022-01-17T21:28:03.029579Z 2022-01-17T21:28:02.753155Z 2022-01-17T21:28:02.548076Z 2022-01-17T21:28:02.138200Z 2022-01-17T21:28:01.844521Z 2022-01-17T21:28:01.789532Z 2022-01-17T21:28:01.201773Z 2022-01-17T21:28:00.205559Z 2022-01-17T21:28:00.141543Z 2022-01-17T21:27:59.767213Z 2022-01-17T21:27:59.608785Z 2022-01-17T21:27:59.399292Z 2022-01-17T21:27:59.040667Z 2022-01-17T21:27:58.812264Z 2022-01-17T21:27:58.780846Z 2022-01-17T21:27:58.515178Z 2022-01-17T21:27:58.474553Z 2022-01-17T21:27:53.278147Z 2022-01-17T21:27:52.599453Z 2022-01-17T21:27:52.300681Z 2022-01-17T21:27:51.670262Z 2022-01-17T21:27:51.985945Z 2022-01-17T21:27:50.703600Z 2022-01-17T21:27:50.964322Z 2022-01-17T21:27:49.197181Z 2022-01-17T21:27:44.891380Z 2022-01-17T21:27:44.138125Z 2022-01-17T21:27:43.304103Z 2022-01-17T21:27:43.203149Z 2022-01-17T21:27:43.118396Z 2022-01-17T21:32:43.330852Z 2022-01-17T21:32:42.454401Z 2022-01-17T21:32:42.729197Z 2022-01-17T21:32:42.098037Z 2022-01-17T21:32:41.204413Z 2022-01-17T21:32:40.585123Z 2022-01-17T21:32:41.137738Z 2022-01-17T21:32:39.166213Z 2022-01-17T21:32:40.317280Z 2022-01-17T21:32:39.804829Z 2022-01-17T21:32:38.743107Z 2022-01-17T21:32:39.031310Z 2022-01-17T21:32:38.431282Z 2022-01-17T21:32:37.875995Z 2022-01-17T21:32:37.096848Z 2022-01-17T21:32:36.826404Z 2022-01-17T21:32:36.346882Z 2022-01-17T21:32:36.235394Z 2022-01-17T21:32:35.591798Z 2022-01-17T21:32:35.198142Z 2022-01-17T21:32:34.822749Z 2022-01-17T21:32:34.651068Z 2022-01-17T21:32:34.187036Z 2022-01-17T21:32:33.445136Z 2022-01-17T21:32:32.916528Z 2022-01-17T21:32:33.639205Z 2022-01-17T21:32:33.408598Z 2022-01-17T21:32:32.164397Z 2022-01-17T21:32:32.511002Z 2022-01-17T21:32:31.354922Z 2022-01-17T21:32:30.804774Z 2022-01-17T21:32:30.551755Z 2022-01-17T21:32:30.159436Z 2022-01-17T21:32:29.698972Z 2022-01-17T21:32:29.340837Z 2022-01-17T21:32:29.435646Z 2022-01-17T21:32:28.845875Z 2022-01-17T21:32:28.561001Z 2022-01-17T21:32:28.354579Z 2022-01-17T21:32:28.004548Z 2022-01-17T21:32:26.632234Z 2022-01-17T21:32:27.070447Z 2022-01-17T21:32:26.281918Z 2022-01-17T21:32:25.731772Z 2022-01-17T21:32:25.452503Z 2022-01-17T21:32:25.109741Z 2022-01-17T21:32:24.898616Z 2022-01-17T21:32:24.462251Z 2022-01-17T21:32:24.405985Z 2022-01-17T21:32:23.729316Z 2022-01-17T21:32:23.471743Z 2022-01-17T21:32:23.304177Z 2022-01-17T21:32:23.140279Z 2022-01-17T21:32:22.918659Z 2022-01-17T21:32:22.573047Z 2022-01-17T21:32:22.234427Z 2022-01-17T21:32:22.205984Z 2022-01-17T21:32:21.810209Z 2022-01-17T21:32:21.731625Z 2022-01-17T21:32:21.233426Z 2022-01-17T21:32:21.058725Z 2022-01-17T21:32:11.431216Z 2022-01-17T21:32:20.276202Z 2022-01-17T21:32:11.217669Z 2022-01-17T21:32:20.170591Z 2022-01-17T21:32:10.664702Z 2022-01-17T21:32:10.729719Z 2022-01-17T21:32:10.170865Z 2022-01-17T21:32:19.770950Z 2022-01-17T21:32:09.580284Z 2022-01-17T21:32:09.333093Z 2022-01-17T21:32:09.130218Z 2022-01-17T21:32:08.946877Z 2022-01-17T21:32:08.433399Z 2022-01-17T21:32:08.193932Z 2022-01-17T21:32:07.784947Z 2022-01-17T21:32:07.514851Z 2022-01-17T21:32:06.877229Z 2022-01-17T21:32:06.552611Z 2022-01-17T21:32:06.180130Z 2022-01-17T21:32:05.940867Z 2022-01-17T21:32:05.892144Z 2022-01-17T21:32:05.443335Z 2022-01-17T21:32:05.465424Z 2022-01-17T21:32:05.079838Z 2022-01-17T21:32:05.000861Z 2022-01-17T21:32:04.600826Z 2022-01-17T21:32:04.378922Z 2022-01-17T21:32:04.252407Z 2022-01-17T21:32:04.197952Z 2022-01-17T21:32:04.039047Z 2022-01-17T21:31:55.817767Z 2022-01-17T21:31:55.318988Z 2022-01-17T21:31:54.510793Z 2022-01-17T21:31:54.123815Z 2022-01-17T21:31:52.322024Z 2022-01-17T21:31:52.049511Z 2022-01-17T21:31:51.983451Z 2022-01-17T21:31:51.411076Z 2022-01-17T21:31:51.331904Z", + "started_diff": 53.9133276, + "successful_job_ids": " [99, 97, 96, 95, 94, 93, 91, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 76, 75, 74, 73, 72, 71, 70, 69, 67, 66, 65, 64, 63, 62, 61, 60, 173, 172, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, 160, 159, 158, 156, 155, 153, 152, 151, 150, 149, 148, 147, 146, 145, 142, 141, 139, 138, 137, 135, 134, 133, 132, 131, 130, 129, 128, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 101, 100, 273, 272, 271, 270, 269, 268, 267, 266, 265, 264, 263, 262, 261, 260, 259, 258, 257, 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, 176, 175, 174, 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, 274, 473, 472, 471, 470, 469, 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, 446, 445, 444, 443, 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, 573, 572, 571, 570, 569, 568, 567, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, 515, 514, 513, 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, 481, 480, 479, 478, 477, 476, 475, 474]" + }, + "started": "2022-01-17T21:58:14.251159+00:00", + "version": 1 +} \ No newline at end of file diff --git a/DELME/tests/tmp2r_yzuvx b/DELME/tests/tmp2r_yzuvx new file mode 100644 index 00000000..18d7acf5 --- /dev/null +++ b/DELME/tests/tmp2r_yzuvx @@ -0,0 +1 @@ +{"foo": "bar"} \ No newline at end of file diff --git a/core/opl/__init__.py b/DELME/tests/tmptcf1dcse similarity index 100% rename from core/opl/__init__.py rename to DELME/tests/tmptcf1dcse diff --git a/DELME/tests/tmpzwpg4z99 b/DELME/tests/tmpzwpg4z99 new file mode 100644 index 00000000..18d7acf5 --- /dev/null +++ b/DELME/tests/tmpzwpg4z99 @@ -0,0 +1 @@ +{"foo": "bar"} \ No newline at end of file diff --git a/DELME/venv_core/bin/Activate.ps1 b/DELME/venv_core/bin/Activate.ps1 new file mode 100644 index 00000000..b49d77ba --- /dev/null +++ b/DELME/venv_core/bin/Activate.ps1 @@ -0,0 +1,247 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } + $env:VIRTUAL_ENV_PROMPT = $Prompt +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/DELME/venv_core/bin/activate b/DELME/venv_core/bin/activate new file mode 100644 index 00000000..14f68440 --- /dev/null +++ b/DELME/venv_core/bin/activate @@ -0,0 +1,69 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/home/jhutar/Checkouts/opl/venv_core" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(venv_core) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(venv_core) " + export VIRTUAL_ENV_PROMPT +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/DELME/venv_core/bin/activate.csh b/DELME/venv_core/bin/activate.csh new file mode 100644 index 00000000..c66b7c09 --- /dev/null +++ b/DELME/venv_core/bin/activate.csh @@ -0,0 +1,26 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/home/jhutar/Checkouts/opl/venv_core" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(venv_core) $prompt" + setenv VIRTUAL_ENV_PROMPT "(venv_core) " +endif + +alias pydoc python -m pydoc + +rehash diff --git a/DELME/venv_core/bin/activate.fish b/DELME/venv_core/bin/activate.fish new file mode 100644 index 00000000..7184586b --- /dev/null +++ b/DELME/venv_core/bin/activate.fish @@ -0,0 +1,69 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + set -e _OLD_FISH_PROMPT_OVERRIDE + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + end + + set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/home/jhutar/Checkouts/opl/venv_core" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "(venv_core) " (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + set -gx VIRTUAL_ENV_PROMPT "(venv_core) " +end diff --git a/DELME/venv_core/bin/cluster_read.py b/DELME/venv_core/bin/cluster_read.py new file mode 100755 index 00000000..89bebf35 --- /dev/null +++ b/DELME/venv_core/bin/cluster_read.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','cluster_read.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'cluster_read.py')()) diff --git a/DELME/venv_core/bin/deep b/DELME/venv_core/bin/deep new file mode 100755 index 00000000..97691165 --- /dev/null +++ b/DELME/venv_core/bin/deep @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from deepdiff.commands import cli +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli()) diff --git a/DELME/venv_core/bin/f2py b/DELME/venv_core/bin/f2py new file mode 100755 index 00000000..e3cf1f65 --- /dev/null +++ b/DELME/venv_core/bin/f2py @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/f2py3 b/DELME/venv_core/bin/f2py3 new file mode 100755 index 00000000..e3cf1f65 --- /dev/null +++ b/DELME/venv_core/bin/f2py3 @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/f2py3.11 b/DELME/venv_core/bin/f2py3.11 new file mode 100755 index 00000000..e3cf1f65 --- /dev/null +++ b/DELME/venv_core/bin/f2py3.11 @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/futurize b/DELME/venv_core/bin/futurize new file mode 100755 index 00000000..0667240f --- /dev/null +++ b/DELME/venv_core/bin/futurize @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from libfuturize.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/jp.py b/DELME/venv_core/bin/jp.py new file mode 100755 index 00000000..be5446c2 --- /dev/null +++ b/DELME/venv_core/bin/jp.py @@ -0,0 +1,54 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python + +import sys +import json +import argparse +from pprint import pformat + +import jmespath +from jmespath import exceptions + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('expression') + parser.add_argument('-f', '--filename', + help=('The filename containing the input data. ' + 'If a filename is not given then data is ' + 'read from stdin.')) + parser.add_argument('--ast', action='store_true', + help=('Pretty print the AST, do not search the data.')) + args = parser.parse_args() + expression = args.expression + if args.ast: + # Only print the AST + expression = jmespath.compile(args.expression) + sys.stdout.write(pformat(expression.parsed)) + sys.stdout.write('\n') + return 0 + if args.filename: + with open(args.filename, 'r') as f: + data = json.load(f) + else: + data = sys.stdin.read() + data = json.loads(data) + try: + sys.stdout.write(json.dumps( + jmespath.search(expression, data), indent=4, ensure_ascii=False)) + sys.stdout.write('\n') + except exceptions.ArityError as e: + sys.stderr.write("invalid-arity: %s\n" % e) + return 1 + except exceptions.JMESPathTypeError as e: + sys.stderr.write("invalid-type: %s\n" % e) + return 1 + except exceptions.UnknownFunctionError as e: + sys.stderr.write("unknown-function: %s\n" % e) + return 1 + except exceptions.ParseError as e: + sys.stderr.write("syntax-error: %s\n" % e) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/DELME/venv_core/bin/junit_cli.py b/DELME/venv_core/bin/junit_cli.py new file mode 100755 index 00000000..b7b20753 --- /dev/null +++ b/DELME/venv_core/bin/junit_cli.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','junit_cli.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'junit_cli.py')()) diff --git a/DELME/venv_core/bin/junitparser b/DELME/venv_core/bin/junitparser new file mode 100755 index 00000000..3a3af563 --- /dev/null +++ b/DELME/venv_core/bin/junitparser @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from junitparser.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/normalizer b/DELME/venv_core/bin/normalizer new file mode 100755 index 00000000..c383d61e --- /dev/null +++ b/DELME/venv_core/bin/normalizer @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from charset_normalizer.cli.normalizer import cli_detect +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli_detect()) diff --git a/DELME/venv_core/bin/pass_or_fail.py b/DELME/venv_core/bin/pass_or_fail.py new file mode 100755 index 00000000..a3e10829 --- /dev/null +++ b/DELME/venv_core/bin/pass_or_fail.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','pass_or_fail.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'pass_or_fail.py')()) diff --git a/DELME/venv_core/bin/pasteurize b/DELME/venv_core/bin/pasteurize new file mode 100755 index 00000000..95cd3ee5 --- /dev/null +++ b/DELME/venv_core/bin/pasteurize @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from libpasteurize.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/pip b/DELME/venv_core/bin/pip new file mode 100755 index 00000000..36a94564 --- /dev/null +++ b/DELME/venv_core/bin/pip @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/pip3 b/DELME/venv_core/bin/pip3 new file mode 100755 index 00000000..36a94564 --- /dev/null +++ b/DELME/venv_core/bin/pip3 @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/pip3.11 b/DELME/venv_core/bin/pip3.11 new file mode 100755 index 00000000..36a94564 --- /dev/null +++ b/DELME/venv_core/bin/pip3.11 @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/DELME/venv_core/bin/python b/DELME/venv_core/bin/python new file mode 120000 index 00000000..acd4152a --- /dev/null +++ b/DELME/venv_core/bin/python @@ -0,0 +1 @@ +/usr/bin/python \ No newline at end of file diff --git a/DELME/venv_core/bin/python3 b/DELME/venv_core/bin/python3 new file mode 120000 index 00000000..d8654aa0 --- /dev/null +++ b/DELME/venv_core/bin/python3 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/DELME/venv_core/bin/python3.11 b/DELME/venv_core/bin/python3.11 new file mode 120000 index 00000000..d8654aa0 --- /dev/null +++ b/DELME/venv_core/bin/python3.11 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/DELME/venv_core/bin/rp_updater.py b/DELME/venv_core/bin/rp_updater.py new file mode 100755 index 00000000..f35c0725 --- /dev/null +++ b/DELME/venv_core/bin/rp_updater.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','rp_updater.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'rp_updater.py')()) diff --git a/DELME/venv_core/bin/status_data.py b/DELME/venv_core/bin/status_data.py new file mode 100755 index 00000000..a08bf668 --- /dev/null +++ b/DELME/venv_core/bin/status_data.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','status_data.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'status_data.py')()) diff --git a/DELME/venv_core/bin/status_data_diff.py b/DELME/venv_core/bin/status_data_diff.py new file mode 100755 index 00000000..0f8a4982 --- /dev/null +++ b/DELME/venv_core/bin/status_data_diff.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','status_data_diff.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'status_data_diff.py')()) diff --git a/DELME/venv_core/bin/status_data_report.py b/DELME/venv_core/bin/status_data_report.py new file mode 100755 index 00000000..e57c1dba --- /dev/null +++ b/DELME/venv_core/bin/status_data_report.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','status_data_report.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'status_data_report.py')()) diff --git a/DELME/venv_core/bin/status_data_updater.py b/DELME/venv_core/bin/status_data_updater.py new file mode 100755 index 00000000..4d7918f7 --- /dev/null +++ b/DELME/venv_core/bin/status_data_updater.py @@ -0,0 +1,33 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# EASY-INSTALL-ENTRY-SCRIPT: 'opl-rhcloud-perf-team-core','console_scripts','status_data_updater.py' +import re +import sys + +# for compatibility with easy_install; see #2198 +__requires__ = 'opl-rhcloud-perf-team-core' + +try: + from importlib.metadata import distribution +except ImportError: + try: + from importlib_metadata import distribution + except ImportError: + from pkg_resources import load_entry_point + + +def importlib_load_entry_point(spec, group, name): + dist_name, _, _ = spec.partition('==') + matches = ( + entry_point + for entry_point in distribution(dist_name).entry_points + if entry_point.group == group and entry_point.name == name + ) + return next(matches).load() + + +globals().setdefault('load_entry_point', importlib_load_entry_point) + + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(load_entry_point('opl-rhcloud-perf-team-core', 'console_scripts', 'status_data_updater.py')()) diff --git a/DELME/venv_core/bin/tabulate b/DELME/venv_core/bin/tabulate new file mode 100755 index 00000000..acd384a0 --- /dev/null +++ b/DELME/venv_core/bin/tabulate @@ -0,0 +1,8 @@ +#!/home/jhutar/Checkouts/opl/venv_core/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from tabulate import _main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(_main()) diff --git a/DELME/venv_core/lib64 b/DELME/venv_core/lib64 new file mode 120000 index 00000000..7951405f --- /dev/null +++ b/DELME/venv_core/lib64 @@ -0,0 +1 @@ +lib \ No newline at end of file diff --git a/DELME/venv_core/pyvenv.cfg b/DELME/venv_core/pyvenv.cfg new file mode 100644 index 00000000..993bb6be --- /dev/null +++ b/DELME/venv_core/pyvenv.cfg @@ -0,0 +1,5 @@ +home = /usr/bin +include-system-site-packages = false +version = 3.11.3 +executable = /usr/bin/python3.11 +command = /usr/bin/python -m venv /home/jhutar/Checkouts/opl/venv_core diff --git a/DELME/venv_core/src/opl-rhcloud-perf-team-core b/DELME/venv_core/src/opl-rhcloud-perf-team-core new file mode 160000 index 00000000..5fbb2fc0 --- /dev/null +++ b/DELME/venv_core/src/opl-rhcloud-perf-team-core @@ -0,0 +1 @@ +Subproject commit 5fbb2fc064403b027cdf3979398b3cd801872f55 diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 00000000..03f19521 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,20 @@ +## AI Software Architect Framework + +This project uses the AI Software Architect framework for architectural decision tracking and reviews. + +### Available Commands + +- **Create ADR**: "Create ADR for [decision]" +- **Architecture Review**: "Start architecture review for version X.Y.Z" +- **Specialist Review**: "Ask [specialist role] to review [target]" +- **Implementation**: "Implement [feature] as the architects" +- **List Members**: "List architecture members" +- **Status**: "What's our architecture status?" + +### Documentation + +All architecture documentation is in `.architecture/`: +- **ADRs**: `.architecture/decisions/adrs/` +- **Reviews**: `.architecture/reviews/` +- **Principles**: `.architecture/principles.md` +- **Team**: `.architecture/members.yml` diff --git a/README.md b/README.md index 40a8b92e..77e27dab 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,14 @@ OPL library Contains various helpers for not only RHCloud performance testing. +Repository Structure +-------------------- + +This repository uses a split-directory workspace layout to cleanly separate lightweight core tools from those requiring heavy dependencies (like `kafka-python`, `psycopg2-binary`, `locust`). Python's implicit namespace packages (PEP 420) are used so that imports like `import opl.some_module` work seamlessly across both directories. + +* `core/opl/` - Contains the lightweight, minimal-dependency core tools. +* `extras/opl/` - Contains tools and generators that require heavier dependencies. + Generic command-line tools -------------------------- @@ -16,11 +24,11 @@ There is couple of tools used mostly to integrate with CPT and comp.: Grafana. It is used by `status_data.py` to enritch status data files with facts (as defined in per-test config file). Usually it is info about the cluster (using `oc ... | jq ...` commands) or monitoring data from - Prometheus. E.g. see `opl/cluster_read_example.yaml`. + Prometheus. E.g. see `core/opl/cluster_read_example.yaml`. * `pass_or_fail.py` - Tool which uses simple statistic in an attempt to decide if latest test result is PASS or FAIL when compared to historical results. Example of the config file is in - `opl/investigator/sample_config.yaml`. + `core/opl/investigator/sample_config.yaml`. * `junit_cli.py` - Script to manipulate JUnit XML file from CLI. It is used by Satellite CPT. * `status_data.py` - Library (and command-line tool) to manipulate status @@ -29,7 +37,7 @@ There is couple of tools used mostly to integrate with CPT and comp.: status data file. It is used by tests to provide final summary without need to write per-test tool which would do it - Jinja2 template seems to provide enough logick for what is needed in the output. Each test have its own - template - e.g. see `opl/status_data_report.txt`. + template - e.g. see `core/opl/status_data_report.txt`. * `status_data_diff.py` - Convenience tool to show differences in two status data files. @@ -40,26 +48,55 @@ These tools are only used by tests: simplest way did not worked for me) * `script-manage-db.py` - Helper script to manipulate with utility DB. Consumes config with list of tables relevant for given test and SQL needed - to create them. E.g. see `opl/sample-tables.yaml` + to create them. E.g. see `extras/opl/sample-tables.yaml` Installation ------------ -Install with: +You can install the full library or just the lightweight core tools. + +**Full Installation:** + +Installs all tools, including those with heavy dependencies. python -m venv venv source venv/bin/activate python -m pip install git+https://github.com/redhat-performance/opl.git -If you have cloned the git and want to develop locally, replace last step with: +**Core Installation:** + +Installs only the lightweight core tools from `core/opl/` to minimize dependency footprint. + + python -m venv venv + source venv/bin/activate + python3 -m pip install --no-cache-dir -e "git+https://github.com/redhat-performance/opl.git#egg=opl-rhcloud-perf-team-core&subdirectory=core" + +**Extras Installation:** + +Note: Do you really need to do this? If you do full installation, it does exactly this. + +If you choose to install only the extras package manually, it is your +responsibility to also install the core package, as extras modules depend on +core functionality (because setting that dependency for this package is not +straight forward): - python -m pip install --editable .[dev] + python3 -m pip install "git+https://github.com/redhat-performance/opl.git#egg=opl-rhcloud-perf-team-core&subdirectory=core" + python3 -m pip install "git+https://github.com/redhat-performance/opl.git#egg=opl-rhcloud-perf-team-extras&subdirectory=extras" + +**Local Development:** + +If you have cloned the git repository and want to develop locally, install the full package in editable mode: + + python -m pip install -e .[dev] Running unit tests ------------------ source venv/bin/activate - python -m pytest + python -m pytest tests/ + +This will also generate a test coverage report. The terminal will show a summary, +and a detailed HTML report can be found in `htmlcov/index.html`. Notes ------------------ diff --git a/core/opl/cluster_read.py b/core/opl/cluster_read.py index 73fd30d7..70ae3eb4 100755 --- a/core/opl/cluster_read.py +++ b/core/opl/cluster_read.py @@ -195,7 +195,14 @@ def _sanitize_target(self, target): return target @retry.retry_on_traceback(max_attempts=10, wait_seconds=1) - def measure(self, ri, name, grafana_target, grafana_enritchment={}, grafana_include_vars=False): + def measure( + self, + ri, + name, + grafana_target, + grafana_enritchment={}, + grafana_include_vars=False, + ): assert ( ri.start is not None and ri.end is not None ), "We need timerange to approach Grafana" diff --git a/core/opl/investigator/check.py b/core/opl/investigator/check.py index 49a0206d..f9166b2c 100644 --- a/core/opl/investigator/check.py +++ b/core/opl/investigator/check.py @@ -172,6 +172,7 @@ def check_by_stdev_3(data, value): """Checks if the current value is within 2 standard deviations of the mean of previous values""" return _check_by_stdev(data, value, 3) + def check_by_provided_min_max(_, value, provided_min, provided_max): """Checks if the current value is simply withing provided min and max values (historical values not used at all)""" info = collections.OrderedDict( @@ -198,7 +199,9 @@ def check(methods, data, value, description="N/A", verbose=True): method_args = method.get("args", []) result, info = globals()[method_name](data, value, *method_args) results.append(result) - logging.info(f"{method_name}({', '.join([str(i) for i in method_args])}) value {value} returned {'PASS' if result else 'FAIL'}") + logging.info( + f"{method_name}({', '.join([str(i) for i in method_args])}) value {value} returned {'PASS' if result else 'FAIL'}" + ) info_full = collections.OrderedDict() info_full["description"] = description diff --git a/core/opl/investigator/config.py b/core/opl/investigator/config.py index e6dfd359..584a33ce 100644 --- a/core/opl/investigator/config.py +++ b/core/opl/investigator/config.py @@ -169,4 +169,6 @@ def load_config(conf, fp): conf.decisions_es_server_verify = True if conf.decisions_type == "csv": - conf.decisions_filename = data["decisions"].get("file", data["decisions"].get("filename")) + conf.decisions_filename = data["decisions"].get( + "file", data["decisions"].get("filename") + ) diff --git a/core/opl/investigator/csv_decisions.py b/core/opl/investigator/csv_decisions.py index 77019dd5..fe138505 100644 --- a/core/opl/investigator/csv_decisions.py +++ b/core/opl/investigator/csv_decisions.py @@ -11,7 +11,9 @@ def store(filename, decisions): for decision in decisions: decision["job_name"] = job_name decision["build_url"] = build_url - decision["uploaded"] = datetime.datetime.now(tz=datetime.timezone.utc).isoformat() + decision["uploaded"] = datetime.datetime.now( + tz=datetime.timezone.utc + ).isoformat() fieldnames = [] for d in decisions: diff --git a/core/opl/junit_cli.py b/core/opl/junit_cli.py index 570555d5..5d6ed835 100755 --- a/core/opl/junit_cli.py +++ b/core/opl/junit_cli.py @@ -10,7 +10,7 @@ import requests -from . import date +from opl import date def now(): diff --git a/core/opl/pass_or_fail.py b/core/opl/pass_or_fail.py index 31c2813a..355682a0 100755 --- a/core/opl/pass_or_fail.py +++ b/core/opl/pass_or_fail.py @@ -14,7 +14,6 @@ import tabulate - STATUSES = { 0: "PASS", 1: "FAIL", @@ -67,7 +66,7 @@ def doit(args): # Render what needs to be rendered to finish config loading opl.investigator.config.load_config_finish(args, current_sd) - sets_list = [s['name'] for s in args.sets] + sets_list = [s["name"] for s in args.sets] # Load data items from current data current = opl.investigator.status_data_loader.load_data(current_sd, sets_list) diff --git a/core/opl/shovel.py b/core/opl/shovel.py index 00e654f6..28675b10 100755 --- a/core/opl/shovel.py +++ b/core/opl/shovel.py @@ -90,7 +90,7 @@ class pluginBase: def __init__(self): self.logger = logging.getLogger(str(self.__class__)) - def set_args(parser, subparsers): + def set_args(self, parser, subparsers): pass @@ -124,9 +124,7 @@ def download(self, args): try: data = response.json() except requests.exceptions.JSONDecodeError: - self.logger.error( - "Failed to parse JSON, ignoring --record-link option" - ) + self.logger.error("Failed to parse JSON, ignoring --record-link option") else: _set_field_value(args.record_link, from_url, data) response_content = str.encode( @@ -354,7 +352,7 @@ def upload(self, args): try: marker = _get_field_value(args.matcher_field, run_data) except KeyError: - pass # If matcher was not found in data, we can assume this is not a duplicate + pass # If matcher was not found in data, we can assume this is not a duplicate else: if marker == matcher_value: print( @@ -533,7 +531,9 @@ def schema_label_list(self, args): labels = self._schema_id_labels(args, schema_id) for label in labels: - print(f"{label['id']}\t{label['name']}\t{label['extractors'][0]['jsonpath']}") + print( + f"{label['id']}\t{label['name']}\t{label['extractors'][0]['jsonpath']}" + ) def schema_label_add(self, args): self._setup(args) @@ -615,7 +615,9 @@ def schema_label_update(self, args): labels = self._schema_id_labels(args, schema_id) if args.update_by_id is not None: - label = next((item for item in labels if item["id"] == args.update_by_id), False) + label = next( + (item for item in labels if item["id"] == args.update_by_id), False + ) if label: # Label with provided id found, lets update it @@ -633,7 +635,9 @@ def schema_label_update(self, args): else: # Label not found, shall we fail now? if args.add_if_missing: - self.logger.warning(f"Label with name {new_name} not found, so adding new one with new ID") + self.logger.warning( + f"Label with name {new_name} not found, so adding new one with new ID" + ) return self.schema_label_add(args) else: raise KeyError(f"Failed to find label with name {new_name}") @@ -641,7 +645,9 @@ def schema_label_update(self, args): raise Exception("Either --update-by-id or --update-by-name have to be used") if new == label: - self.logger.info(f"Proposed and current label {label['id']} in schema ID {schema_id} are same, nothing to change") + self.logger.info( + f"Proposed and current label {label['id']} in schema ID {schema_id} are same, nothing to change" + ) else: self.logger.debug(f"Updating label in schema id {schema_id}: {new}") response = requests.put( @@ -758,9 +764,7 @@ def set_args(self, parser, subparsers): ) # Options for listing results - subparser = subparsers.add_parser( - "list", help="List test run IDs for a test" - ) + subparser = subparsers.add_parser("list", help="List test run IDs for a test") subparser.set_defaults(func=self.list) subparser.add_argument( "--test-name", @@ -791,9 +795,7 @@ def set_args(self, parser, subparsers): ) # Options for adding schema label - subparser = subparsers.add_parser( - "schema-label-add", help="Add schema label" - ) + subparser = subparsers.add_parser("schema-label-add", help="Add schema label") subparser.set_defaults(func=self.schema_label_add) subparser.add_argument( "--schema-uri", @@ -1113,7 +1115,9 @@ def links(self, args): # Make all the links absolute to initial user provided URL. # If there is already absolute link like 'http://www.example.com', # it will not be affected - absolute_hrefs = [urllib.parse.urljoin(args.url, href) for href in filtered_hrefs] + absolute_hrefs = [ + urllib.parse.urljoin(args.url, href) for href in filtered_hrefs + ] # Print what we found for href in absolute_hrefs: diff --git a/core/opl/skelet.py b/core/opl/skelet.py index f83700c7..85a66e95 100644 --- a/core/opl/skelet.py +++ b/core/opl/skelet.py @@ -4,7 +4,7 @@ import time from contextlib import contextmanager -from . import status_data +from opl import status_data def setup_logger(app_name, stderr_log_lvl): @@ -50,7 +50,7 @@ def setup_logger(app_name, stderr_log_lvl): @contextmanager -def test_setup(parser, logger_name="root"): +def test_setup(parser, logger_name="root", args=None): parser.add_argument( "--status-data-file", default=os.getenv("STATUS_DATA_FILE", "/tmp/status-data.json"), @@ -68,20 +68,20 @@ def test_setup(parser, logger_name="root"): action="store_true", help="Show debug output", ) - args = parser.parse_args() + parsed_args = parser.parse_args(args) - if args.debug: + if parsed_args.debug: logger = setup_logger(logger_name, logging.DEBUG) - elif args.verbose: + elif parsed_args.verbose: logger = setup_logger(logger_name, logging.INFO) else: logger = setup_logger(logger_name, logging.WARNING) - logger.debug(f"Args: {args}") + logger.debug(f"Args: {parsed_args}") - sdata = status_data.StatusData(args.status_data_file) + sdata = status_data.StatusData(parsed_args.status_data_file) try: - yield (args, sdata) + yield (parsed_args, sdata) finally: sdata.save() diff --git a/core/opl/status_data.py b/core/opl/status_data.py index 102450df..b17fb1e0 100755 --- a/core/opl/status_data.py +++ b/core/opl/status_data.py @@ -18,9 +18,9 @@ import yaml -from . import cluster_read -from . import date -from . import skelet +from opl import cluster_read +from opl import date +from opl import skelet class StatusData: @@ -143,11 +143,15 @@ def _set(self, data, split_key, value): # Check that array key is only used if this is last sub-key if array_key: - assert last_key, "Arrays can only be last in the multi keys (i.e. 'aaa.bbb[]', but not 'aaa[]'.bbb)" + assert ( + last_key + ), "Arrays can only be last in the multi keys (i.e. 'aaa.bbb[]', but not 'aaa[]'.bbb)" # Check that we are not attempting to change type of already existing key if array_key and not missing_key: - assert type(data[current_key]) == list, "You are trying to change type (e.g. 'aaa' was string and now you are trying to add to 'aaa[]')" + assert ( + type(data[current_key]) is list + ), "You are trying to change type (e.g. 'aaa' was string and now you are trying to add to 'aaa[]')" if missing_key: if last_key: @@ -155,9 +159,9 @@ def _set(self, data, split_key, value): data[current_key] = [value] else: data[current_key] = value - return # This was last key, we are done + return # This was last key, we are done else: - data[current_key] = {} # This is not last key, so it can not be array + data[current_key] = {} # This is not last key, so it can not be array return self._set(data[current_key], split_key[1:], value) else: if last_key: @@ -165,9 +169,11 @@ def _set(self, data, split_key, value): data[current_key].append(value) else: data[current_key] = value - return # This was last key, we are done + return # This was last key, we are done else: - return self._set(data[current_key], split_key[1:], value) # This is not last key, so no need to check for array + return self._set( + data[current_key], split_key[1:], value + ) # This is not last key, so no need to check for array def set(self, multikey, value): """ diff --git a/core/opl/status_data_updater.py b/core/opl/status_data_updater.py index 38bcdc13..9746b247 100755 --- a/core/opl/status_data_updater.py +++ b/core/opl/status_data_updater.py @@ -21,7 +21,6 @@ import yaml - RP_TO_ES_STATE = { "automation_bug": "FAIL", "no_defect": "PASS", diff --git a/core/setup.py b/core/setup.py index 934101c5..418bb900 100644 --- a/core/setup.py +++ b/core/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import setuptools with open("../README.md", "r") as fh: @@ -13,8 +11,8 @@ description="Our performance library, core bits", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/TODO/TODO", - packages=setuptools.find_packages(), + url="https://github.com/redhat-performance/opl", + packages=setuptools.find_namespace_packages(include=["opl.*", "opl"]), classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License (GPL)", diff --git a/opl/consumer_lag.py b/extras/opl/consumer_lag.py similarity index 100% rename from opl/consumer_lag.py rename to extras/opl/consumer_lag.py diff --git a/opl/db.py b/extras/opl/db.py similarity index 100% rename from opl/db.py rename to extras/opl/db.py diff --git a/opl/generators/README.md b/extras/opl/generators/README.md similarity index 100% rename from opl/generators/README.md rename to extras/opl/generators/README.md diff --git a/opl/__init__.py b/extras/opl/generators/__init__.py similarity index 100% rename from opl/__init__.py rename to extras/opl/generators/__init__.py diff --git a/opl/generators/chrome_notifications.py b/extras/opl/generators/chrome_notifications.py similarity index 100% rename from opl/generators/chrome_notifications.py rename to extras/opl/generators/chrome_notifications.py diff --git a/opl/generators/chrome_notifications_template.json.j2 b/extras/opl/generators/chrome_notifications_template.json.j2 similarity index 100% rename from opl/generators/chrome_notifications_template.json.j2 rename to extras/opl/generators/chrome_notifications_template.json.j2 diff --git a/opl/generators/create_packages_template_from_dnf_repoquery.py b/extras/opl/generators/create_packages_template_from_dnf_repoquery.py similarity index 100% rename from opl/generators/create_packages_template_from_dnf_repoquery.py rename to extras/opl/generators/create_packages_template_from_dnf_repoquery.py diff --git a/opl/generators/edge_inventory_egress_template.json.j2 b/extras/opl/generators/edge_inventory_egress_template.json.j2 similarity index 100% rename from opl/generators/edge_inventory_egress_template.json.j2 rename to extras/opl/generators/edge_inventory_egress_template.json.j2 diff --git a/opl/generators/edge_inventory_ingress_template.json.j2 b/extras/opl/generators/edge_inventory_ingress_template.json.j2 similarity index 100% rename from opl/generators/edge_inventory_ingress_template.json.j2 rename to extras/opl/generators/edge_inventory_ingress_template.json.j2 diff --git a/opl/generators/enabled_services.txt b/extras/opl/generators/enabled_services.txt similarity index 100% rename from opl/generators/enabled_services.txt rename to extras/opl/generators/enabled_services.txt diff --git a/opl/generators/fifi_results.py b/extras/opl/generators/fifi_results.py similarity index 100% rename from opl/generators/fifi_results.py rename to extras/opl/generators/fifi_results.py diff --git a/opl/generators/generic.py b/extras/opl/generators/generic.py similarity index 100% rename from opl/generators/generic.py rename to extras/opl/generators/generic.py diff --git a/opl/generators/installed_services.txt b/extras/opl/generators/installed_services.txt similarity index 100% rename from opl/generators/installed_services.txt rename to extras/opl/generators/installed_services.txt diff --git a/opl/generators/inventory_egress.py b/extras/opl/generators/inventory_egress.py similarity index 100% rename from opl/generators/inventory_egress.py rename to extras/opl/generators/inventory_egress.py diff --git a/opl/generators/inventory_egress_data.json b/extras/opl/generators/inventory_egress_data.json similarity index 100% rename from opl/generators/inventory_egress_data.json rename to extras/opl/generators/inventory_egress_data.json diff --git a/opl/generators/inventory_egress_data_rhel88.json b/extras/opl/generators/inventory_egress_data_rhel88.json similarity index 100% rename from opl/generators/inventory_egress_data_rhel88.json rename to extras/opl/generators/inventory_egress_data_rhel88.json diff --git a/opl/generators/inventory_egress_template.json.j2 b/extras/opl/generators/inventory_egress_template.json.j2 similarity index 100% rename from opl/generators/inventory_egress_template.json.j2 rename to extras/opl/generators/inventory_egress_template.json.j2 diff --git a/opl/generators/inventory_ingress.py b/extras/opl/generators/inventory_ingress.py similarity index 100% rename from opl/generators/inventory_ingress.py rename to extras/opl/generators/inventory_ingress.py diff --git a/opl/generators/inventory_ingress_InvGitUtilsPayload_template.json.j2 b/extras/opl/generators/inventory_ingress_InvGitUtilsPayload_template.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_InvGitUtilsPayload_template.json.j2 rename to extras/opl/generators/inventory_ingress_InvGitUtilsPayload_template.json.j2 diff --git a/opl/generators/inventory_ingress_RHSM_template.json.j2 b/extras/opl/generators/inventory_ingress_RHSM_template.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_RHSM_template.json.j2 rename to extras/opl/generators/inventory_ingress_RHSM_template.json.j2 diff --git a/opl/generators/inventory_ingress_puptoo_template.json.j2 b/extras/opl/generators/inventory_ingress_puptoo_template.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_puptoo_template.json.j2 rename to extras/opl/generators/inventory_ingress_puptoo_template.json.j2 diff --git a/opl/generators/inventory_ingress_puptoo_template_compliance.json.j2 b/extras/opl/generators/inventory_ingress_puptoo_template_compliance.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_puptoo_template_compliance.json.j2 rename to extras/opl/generators/inventory_ingress_puptoo_template_compliance.json.j2 diff --git a/opl/generators/inventory_ingress_puptoo_template_rhel90.json.j2 b/extras/opl/generators/inventory_ingress_puptoo_template_rhel90.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_puptoo_template_rhel90.json.j2 rename to extras/opl/generators/inventory_ingress_puptoo_template_rhel90.json.j2 diff --git a/opl/generators/inventory_ingress_yupana_template.json.j2 b/extras/opl/generators/inventory_ingress_yupana_template.json.j2 similarity index 100% rename from opl/generators/inventory_ingress_yupana_template.json.j2 rename to extras/opl/generators/inventory_ingress_yupana_template.json.j2 diff --git a/opl/generators/notifications_ingres_template.json.j2 b/extras/opl/generators/notifications_ingres_template.json.j2 similarity index 100% rename from opl/generators/notifications_ingres_template.json.j2 rename to extras/opl/generators/notifications_ingres_template.json.j2 diff --git a/opl/generators/notifications_ingress.py b/extras/opl/generators/notifications_ingress.py similarity index 100% rename from opl/generators/notifications_ingress.py rename to extras/opl/generators/notifications_ingress.py diff --git a/opl/generators/package_data_rhel88.json b/extras/opl/generators/package_data_rhel88.json similarity index 100% rename from opl/generators/package_data_rhel88.json rename to extras/opl/generators/package_data_rhel88.json diff --git a/opl/generators/packages.py b/extras/opl/generators/packages.py similarity index 100% rename from opl/generators/packages.py rename to extras/opl/generators/packages.py diff --git a/opl/generators/packages_data.json b/extras/opl/generators/packages_data.json similarity index 100% rename from opl/generators/packages_data.json rename to extras/opl/generators/packages_data.json diff --git a/opl/generators/packages_rhel74.json b/extras/opl/generators/packages_rhel74.json similarity index 100% rename from opl/generators/packages_rhel74.json rename to extras/opl/generators/packages_rhel74.json diff --git a/opl/generators/packages_rhel78.json b/extras/opl/generators/packages_rhel78.json similarity index 100% rename from opl/generators/packages_rhel78.json rename to extras/opl/generators/packages_rhel78.json diff --git a/opl/generators/packages_rhel80.json b/extras/opl/generators/packages_rhel80.json similarity index 100% rename from opl/generators/packages_rhel80.json rename to extras/opl/generators/packages_rhel80.json diff --git a/opl/generators/packages_rhel84.json b/extras/opl/generators/packages_rhel84.json similarity index 100% rename from opl/generators/packages_rhel84.json rename to extras/opl/generators/packages_rhel84.json diff --git a/opl/generators/packages_rhel90.json b/extras/opl/generators/packages_rhel90.json similarity index 100% rename from opl/generators/packages_rhel90.json rename to extras/opl/generators/packages_rhel90.json diff --git a/opl/generators/playbook-dispatcher-post-run.json.j2 b/extras/opl/generators/playbook-dispatcher-post-run.json.j2 similarity index 100% rename from opl/generators/playbook-dispatcher-post-run.json.j2 rename to extras/opl/generators/playbook-dispatcher-post-run.json.j2 diff --git a/opl/generators/playbook-dispatcher-runner-updates.json.j2 b/extras/opl/generators/playbook-dispatcher-runner-updates.json.j2 similarity index 100% rename from opl/generators/playbook-dispatcher-runner-updates.json.j2 rename to extras/opl/generators/playbook-dispatcher-runner-updates.json.j2 diff --git a/opl/generators/playbook_dispatcher.py b/extras/opl/generators/playbook_dispatcher.py similarity index 100% rename from opl/generators/playbook_dispatcher.py rename to extras/opl/generators/playbook_dispatcher.py diff --git a/opl/generators/qpc_tarball.py b/extras/opl/generators/qpc_tarball.py similarity index 98% rename from opl/generators/qpc_tarball.py rename to extras/opl/generators/qpc_tarball.py index 2f9f91fe..395dca9d 100644 --- a/opl/generators/qpc_tarball.py +++ b/extras/opl/generators/qpc_tarball.py @@ -20,7 +20,9 @@ def get_tarball_message(account, remotename, size, download_url): "service": "qpc", "size": size, "url": download_url, - "b64_identity": opl.gen.get_auth_header(account, account).decode("UTF-8"), + "b64_identity": opl.gen.get_auth_header(account, account, account).decode( + "UTF-8" + ), "timestamp": opl.gen.gen_datetime().replace("+00:00", "Z"), } return json.dumps(data) diff --git a/opl/generators/running_processes.txt b/extras/opl/generators/running_processes.txt similarity index 100% rename from opl/generators/running_processes.txt rename to extras/opl/generators/running_processes.txt diff --git a/opl/generators/yum_repos.txt b/extras/opl/generators/yum_repos.txt similarity index 100% rename from opl/generators/yum_repos.txt rename to extras/opl/generators/yum_repos.txt diff --git a/opl/get_db_times.py b/extras/opl/get_db_times.py similarity index 99% rename from opl/get_db_times.py rename to extras/opl/get_db_times.py index 4c3aa733..059dae76 100755 --- a/opl/get_db_times.py +++ b/extras/opl/get_db_times.py @@ -16,7 +16,6 @@ import yaml - """ You want to use this helper if you want to get timestamps on when your hosts landed in application DB and store these timestamps in storage DB. diff --git a/opl/get_kafka_times.py b/extras/opl/get_kafka_times.py similarity index 100% rename from opl/get_kafka_times.py rename to extras/opl/get_kafka_times.py diff --git a/opl/hbi_utils.py b/extras/opl/hbi_utils.py similarity index 97% rename from opl/hbi_utils.py rename to extras/opl/hbi_utils.py index 6b2ddbea..907b7dec 100644 --- a/opl/hbi_utils.py +++ b/extras/opl/hbi_utils.py @@ -138,7 +138,7 @@ def verify(args, previous_records, status_data, inventory, collect_info): break elif existing_ids > expected_ids: logging.warning( - f"We have more hosts than expected! We have {existing_ids-previous_records} of {args.count}" + f"We have more hosts than expected! We have {existing_ids - previous_records} of {args.count}" ) break @@ -146,18 +146,18 @@ def verify(args, previous_records, status_data, inventory, collect_info): attempt += 1 if attempt > attempts_max: raise Exception( - f"After {attempt} attempts, we only have {existing_ids-previous_records} out of {args.count}" + f"After {attempt} attempts, we only have {existing_ids - previous_records} out of {args.count}" ) # If there were no new hosts now, wait a bit if existing_ids != expected_ids: logging.debug( - f"Waiting for IDs, attempt {attempt}, remaining {existing_ids-previous_records} out of {args.count}, in total there are {existing_ids} out of {expected_ids} expected hosts in HBI" + f"Waiting for IDs, attempt {attempt}, remaining {existing_ids - previous_records} out of {args.count}, in total there are {existing_ids} out of {expected_ids} expected hosts in HBI" ) time.sleep(15) elif existing_ids > expected_ids: logging.warning( - f"We have more hosts than expected! We have {existing_ids-previous_records} of {args.count}" + f"We have more hosts than expected! We have {existing_ids - previous_records} of {args.count}" ) inventory_cursor.close() diff --git a/opl/horreum_api.py b/extras/opl/horreum_api.py similarity index 99% rename from opl/horreum_api.py rename to extras/opl/horreum_api.py index ba31f0e2..f8b48415 100755 --- a/opl/horreum_api.py +++ b/extras/opl/horreum_api.py @@ -374,7 +374,7 @@ def update_variables(self, test_id: int, variables: List[Dict[str, Any]]) -> boo has_cd = bool(var.get("changeDetection")) cd_count = len(var.get("changeDetection", [])) logger.info( - f" [{i+1}] {var_name} (changeDetection: {has_cd}, entries: {cd_count})" + f" [{i + 1}] {var_name} (changeDetection: {has_cd}, entries: {cd_count})" ) try: @@ -440,7 +440,7 @@ def create_multiple_change_detection_variables(self, test_id, variables_data): ) for i, var in enumerate(variables_data[:10]): # Show first 10 logger.info( - f" {i+1}. {var.get('name', 'N/A')} (group: {var.get('group', 'N/A')})" + f" {i + 1}. {var.get('name', 'N/A')} (group: {var.get('group', 'N/A')})" ) if len(variables_data) > 10: logger.info(f" ... and {len(variables_data) - 10} more") @@ -1550,7 +1550,7 @@ def main(): if label_name in existing_label_names: skipped_labels.append(label_name) logger.warning( - f"Skipped {i+1}/{len(label_defs)}: {label_name} (already exists)" + f"Skipped {i + 1}/{len(label_defs)}: {label_name} (already exists)" ) continue @@ -1558,10 +1558,12 @@ def main(): try: label = api.create_label(schema_id, label_def) created_labels.append(label) - logger.info(f"Created {i+1}/{len(label_defs)}: {label_name}") + logger.info(f"Created {i + 1}/{len(label_defs)}: {label_name}") except Exception as e: failed_labels.append((label_name, str(e))) - logger.error(f"Failed {i+1}/{len(label_defs)}: {label_name} - {e}") + logger.error( + f"Failed {i + 1}/{len(label_defs)}: {label_name} - {e}" + ) else: if not skip_labels: logger.warning("Skipping label creation - no valid schema ID") diff --git a/opl/http.py b/extras/opl/http.py similarity index 99% rename from opl/http.py rename to extras/opl/http.py index 3e8ce3cd..237d0492 100644 --- a/opl/http.py +++ b/extras/opl/http.py @@ -4,7 +4,6 @@ import urllib3 - session = requests.Session() diff --git a/opl/inventory.py b/extras/opl/inventory.py similarity index 100% rename from opl/inventory.py rename to extras/opl/inventory.py diff --git a/opl/kafka_init.py b/extras/opl/kafka_init.py similarity index 100% rename from opl/kafka_init.py rename to extras/opl/kafka_init.py diff --git a/opl/locust.py b/extras/opl/locust.py similarity index 98% rename from opl/locust.py rename to extras/opl/locust.py index a9998a89..edc8e88c 100644 --- a/opl/locust.py +++ b/extras/opl/locust.py @@ -5,9 +5,9 @@ import gevent -import locust.env -import locust.log -import locust.stats +from locust import env as l_env # pylint: disable=no-name-in-module +from locust import log as l_log # pylint: disable=no-name-in-module +from locust import stats as l_stats # pylint: disable=no-name-in-module import tabulate @@ -42,7 +42,7 @@ def run_locust(args, status_data, test_set, new_stats=False, summary_only=False) f"Running with host = {args.host}, num_clients = {args.num_clients}, hatch_rate = {args.hatch_rate}, duration = {args.test_duration} / requests = {args.test_requests}" ) - env = locust.env.Environment() + env = l_env.Environment() env.user_classes = [test_set] env.stop_timeout = args.stop_timeout env.host = args.host diff --git a/opl/manage_db.py b/extras/opl/manage_db.py similarity index 97% rename from opl/manage_db.py rename to extras/opl/manage_db.py index 86e73336..8d3c685a 100755 --- a/opl/manage_db.py +++ b/extras/opl/manage_db.py @@ -5,12 +5,13 @@ import time import psycopg2 +from psycopg2.errors import UndefinedTable # pylint: disable=no-name-in-module import yaml -from . import args -from . import db -from . import skelet +from opl import args +from opl import db +from opl import skelet def execute_query(connection, query): @@ -81,7 +82,7 @@ def recreate_table(connection, table, table_sql): logging.debug(f"Dropping table {table}") try: cursor.execute(f"DROP TABLE {table}") - except (psycopg2.InternalError, psycopg2.errors.UndefinedTable) as e: + except (psycopg2.InternalError, UndefinedTable) as e: logging.error(f"Failed to drop {table}: {e}") cursor = connection.cursor() connection.commit() diff --git a/opl/post_kafka_times.py b/extras/opl/post_kafka_times.py similarity index 99% rename from opl/post_kafka_times.py rename to extras/opl/post_kafka_times.py index d5fb05ec..a6a49ccc 100644 --- a/opl/post_kafka_times.py +++ b/extras/opl/post_kafka_times.py @@ -18,7 +18,6 @@ import yaml - """ You want to use this helper if you want to achieve this: diff --git a/opl/rbac_populate.py b/extras/opl/rbac_populate.py similarity index 100% rename from opl/rbac_populate.py rename to extras/opl/rbac_populate.py diff --git a/opl/rbac_utils.py b/extras/opl/rbac_utils.py similarity index 82% rename from opl/rbac_utils.py rename to extras/opl/rbac_utils.py index b44cb951..13632d59 100644 --- a/opl/rbac_utils.py +++ b/extras/opl/rbac_utils.py @@ -78,14 +78,14 @@ class TestRequestedInfo(unittest.TestCase): def test_empty(self): data = RbacTestData() - self.assertEquals(data.get_accounts(), []) + self.assertEqual(data.get_accounts(), []) def test_add_get(self): data = RbacTestData() data.add_account("10001", ["aaa", "bbb"], ["xxx", "yyy", "zzz"]) - self.assertEquals(set(data.get_accounts()), set(["10001"])) - self.assertEquals(set(data.get_users_for_account("10001")), set(["aaa", "bbb"])) - self.assertEquals( + self.assertEqual(set(data.get_accounts()), set(["10001"])) + self.assertEqual(set(data.get_users_for_account("10001")), set(["aaa", "bbb"])) + self.assertEqual( set(data.get_applications_for_account("10001")), set(["xxx", "yyy", "zzz"]) ) @@ -93,19 +93,19 @@ def test_add_more(self): data = RbacTestData() data.add_account("10001", ["aaa", "bbb"], ["xxx"]) data.add_account("10002", ["ccc", "ddd"], ["yyy"]) - self.assertEquals(set(data.get_accounts()), set(["10001", "10002"])) - self.assertEquals(set(data.get_users_for_account("10001")), set(["aaa", "bbb"])) - self.assertEquals(set(data.get_users_for_account("10002")), set(["ccc", "ddd"])) + self.assertEqual(set(data.get_accounts()), set(["10001", "10002"])) + self.assertEqual(set(data.get_users_for_account("10001")), set(["aaa", "bbb"])) + self.assertEqual(set(data.get_users_for_account("10002")), set(["ccc", "ddd"])) def test_add_merge(self): data = RbacTestData() data.add_account("10001", ["aaa", "bbb"], ["xxx"]) data.add_account("10001", ["bbb", "ccc"], ["xxx", "yyy", "zzz"]) - self.assertEquals(set(data.get_accounts()), set(["10001"])) - self.assertEquals( + self.assertEqual(set(data.get_accounts()), set(["10001"])) + self.assertEqual( set(data.get_users_for_account("10001")), set(["aaa", "bbb", "ccc"]) ) - self.assertEquals( + self.assertEqual( set(data.get_applications_for_account("10001")), set(["xxx", "yyy", "zzz"]) ) @@ -115,6 +115,6 @@ def test_info(self): data.add_account("10002", ["ccc", "ddd"], ["yyy", "zzz"]) data.add_account("10003", ["eee"], ["zzz"]) info = data.info() - self.assertEquals(info["accounts_count"], 3) - self.assertEquals(info["users_count"], 5) - self.assertEquals(info["applications_count"], 4) + self.assertEqual(info["accounts_count"], 3) + self.assertEqual(info["users_count"], 5) + self.assertEqual(info["applications_count"], 4) diff --git a/opl/s3_tools.py b/extras/opl/s3_tools.py similarity index 100% rename from opl/s3_tools.py rename to extras/opl/s3_tools.py diff --git a/opl/sample-horreum_fields_config.yaml b/extras/opl/sample-horreum_fields_config.yaml similarity index 100% rename from opl/sample-horreum_fields_config.yaml rename to extras/opl/sample-horreum_fields_config.yaml diff --git a/opl/sample-tables.yaml b/extras/opl/sample-tables.yaml similarity index 100% rename from opl/sample-tables.yaml rename to extras/opl/sample-tables.yaml diff --git a/opl/skip_to_end.py b/extras/opl/skip_to_end.py similarity index 95% rename from opl/skip_to_end.py rename to extras/opl/skip_to_end.py index cedde49e..51801f06 100755 --- a/opl/skip_to_end.py +++ b/extras/opl/skip_to_end.py @@ -5,9 +5,9 @@ import os import time -from .kafka_init import kafka_init -from . import args -from . import skelet +from opl.kafka_init import kafka_init +from opl import args +from opl import skelet def doit_seek_to_end(args): diff --git a/extras/setup.py b/extras/setup.py new file mode 100644 index 00000000..a166418f --- /dev/null +++ b/extras/setup.py @@ -0,0 +1,52 @@ +import setuptools + +with open("../README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="opl-rhcloud-perf-team-extras", + version="0.0.1", + maintainer="Jan Hutar", + maintainer_email="jhutar@redhat.com", + description="Our performance library, extra bits", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/redhat-performance/opl", + packages=setuptools.find_namespace_packages(include=["opl.*", "opl"]), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License (GPL)", + "Operating System :: POSIX :: Linux", + "Intended Audience :: Developers", + "Topic :: Software Development :: Quality Assurance", + ], + python_requires=">=3.6", + install_requires=[ + "kafka-python", + "locust", + "psycopg2-binary", + "numpy", + ], + package_data={ + "opl": [ + "*.yaml", + "*.json", + "*.txt", + ], + "opl.generators": [ + "*.json", + "*.j2", + "*.txt", + ], + }, + entry_points={ + "console_scripts": [ + "script-skip-to-end.py = opl.skip_to_end:main", + "script-manage-db.py = opl.manage_db:main", + "script-hbi-populate.py = opl.hbi_utils:populate_main", + "script-hbi-cleanup.py = opl.hbi_utils:cleanup_main", + "create_packages_template_from_dnf_repoquery.py = opl.create_packages_template_from_dnf_repoquery:parse_repoquery_output_from_stdin", + "horreum_api.py = opl.horreum_api:main", + ], + }, +) diff --git a/opl/args.py b/opl/args.py deleted file mode 100644 index 8f042897..00000000 --- a/opl/args.py +++ /dev/null @@ -1,363 +0,0 @@ -import os -import socket -import argparse - - -def _add_generic_db_opts(parser, name): - name_lower = name.lower() - name_upper = name.upper() - name_camel = f"{name_upper[0]}{name_lower[1:]}" - parser.add_argument( - f"--{name_lower}-db-host", - default=os.getenv(f"{name_upper}_DB_HOST", "localhost"), - help=f"{name_camel} DB host (also use env variable {name_upper}_DB_HOST)", - ) - parser.add_argument( - f"--{name_lower}-db-port", - default=os.getenv(f"{name_upper}_DB_PORT", "5432"), - help=f"{name_camel} DB port (also use env variable {name_upper}_DB_PORT)", - ) - parser.add_argument( - f"--{name_lower}-db-name", - default=os.getenv(f"{name_upper}_DB_NAME", name_lower), - help=f"{name_camel} DB database name (also use env variable {name_upper}_DB_NAME)", - ) - parser.add_argument( - f"--{name_lower}-db-user", - default=os.getenv(f"{name_upper}_DB_USER", name_lower), - help=f"{name_camel} DB username (also use env variable {name_upper}_DB_USER)", - ) - parser.add_argument( - f"--{name_lower}-db-pass", - default=os.getenv(f"{name_upper}_DB_PASS", name_lower), - help=f"{name_camel} DB password (also use env variable {name_upper}_DB_PASS)", - ) - - -def add_cyndi_db_opts(parser): - _add_generic_db_opts(parser, "cyndi") - - -def add_edge_db_opts(parser): - _add_generic_db_opts(parser, "edge") - - -def add_storage_db_opts(parser): - _add_generic_db_opts(parser, "storage") - - -def add_subscriptions_db_opts(parser): - _add_generic_db_opts(parser, "subscriptions") - - -def add_inventory_db_opts(parser): - _add_generic_db_opts(parser, "inventory") - - -def add_patchman_db_opts(parser): - _add_generic_db_opts(parser, "patchman") - - -def add_rbac_db_opts(parser): - _add_generic_db_opts(parser, "rbac") - - -def add_sources_db_opts(parser): - _add_generic_db_opts(parser, "sources") - - -def add_notifications_db_opts(parser): - _add_generic_db_opts(parser, "notifications") - - -def add_rhsm_db_opts(parser): - _add_generic_db_opts(parser, "rhsm") - - -def add_remediations_db_opts(parser): - _add_generic_db_opts(parser, "remediations") - - -def add_vulnerability_db_opts(parser): - _add_generic_db_opts(parser, "vulnerability") - - -def add_ros_db_opts(parser): - _add_generic_db_opts(parser, "ros") - - -def add_compliance_db_opts(parser): - _add_generic_db_opts(parser, "compliance") - - -def add_export_db_opts(parser): - _add_generic_db_opts(parser, "export") - - -def add_playbook_db_opts(parser): - _add_generic_db_opts(parser, "playbook") - - -def add_kafka_opts(parser): - parser.add_argument( - "--kafka-host", - default=os.getenv("KAFKA_HOST", "localhost"), - help="Kafka host (also use env variable KAFKA_HOST). Can get overriden by --kafka-hosts arg or KAFKA_HOSTS envvar.", - ) - parser.add_argument( - "--kafka-hosts", - default=os.getenv("KAFKA_HOSTS", ""), - help="Comma-separated list of hosts, including their ports (also use env variable KAFKA_HOSTS). Takes precedence over --kafka-host and --kafka-port or their envvar variants.", - ) - parser.add_argument( - "--kafka-port", - type=int, - default=int(os.getenv("KAFKA_PORT", 9092)), - help="Kafka port (also use env variable KAFKA_PORT)", - ) - parser.add_argument( - "--kafka-acks", - default=os.getenv("KAFKA_ACKS", "all"), - help="How many acknowledgments the producer requires, either all, 1 or 0 (also use env variable KAFKA_ACKS)", - ) - parser.add_argument( - "--kafka-timeout", - type=int, - default=int(os.getenv("KAFKA_TIMEOUT", 100000)), - help="Kafka timeout when consuming messages (also use env variable KAFKA_TIMEOUT)", - ) - parser.add_argument( - "--kafka-group", - default=os.getenv("KAFKA_GROUP", f"perf-test-{socket.gethostname()}"), - help="Kafka consumer group (also use env variable KAFKA_GROUP)", - ) - parser.add_argument( - "--kafka-username", - default=os.getenv("KAFKA_USERNAME", ""), - help="Kafka username when logging into SASL cluster like MSK (also use env variable KAFKA_USERNAME)", - ) - parser.add_argument( - "--kafka-password", - default=os.getenv("KAFKA_PASSWORD", ""), - help="Kafka password when logging into SASL cluster like MSK (also use env variable KAFKA_PASSWORD)", - ) - parser.add_argument( - "--kafka-request-timeout-ms", - type=int, - default=int(os.getenv("KAFKA_REQUEST_TIMEOUT_MS", 30000)), - help="The client is going to wait this much time for the server to respond to a request (also use env variable KAFKA_REQUEST_TIMEOUT_MS)", - ) - parser.add_argument( - "--kafka-max-block-ms", - type=int, - default=int(os.getenv("KAFKA_MAX_BLOCK_MS", 60000)), - help="Max time to block send e.g. because buffer is full (also use env variable KAFKA_MAX_BLOCK_MS)", - ) - parser.add_argument( - "--kafka-linger-ms", - type=int, - default=int(os.getenv("KAFKA_LINGER_MS", 0)), - help="Max time to wait for more messages when creating batch (also use env variable KAFKA_LINGER_MS)", - ) - parser.add_argument( - "--kafka-compression-type", - choices=[None, "gzip", "snappy", "lz4"], - default=os.getenv("KAFKA_COMPRESSION_TYPE", None), - help="The compression type for all data generated by the producer (also use env variable KAFKA_COMPRESSION_TYPE)", - ) - parser.add_argument( - "--kafka-batch-size", - type=int, - default=int(os.getenv("KAFKA_BATCH_SIZE", 16384)), - help="Max size of the batch before sending (also use env variable KAFKA_BATCH_SIZE)", - ) - parser.add_argument( - "--kafka-buffer-memory", - type=int, - default=int(os.getenv("KAFKA_BUFFER_MEMORY", 33554432)), - help="Memory the producer can use at max for batching (also use env variable KAFKA_BUFFER_MEMORY)", - ) - parser.add_argument( - "--kafka-retries", - type=int, - default=int(os.getenv("KAFKA_RETRIES", 0)), - help="Resend any record whose send fails this many times. Can cause duplicates! (also use env variable KAFKA_RETRIES)", - ) - - -def add_mosquitto_opts(parser): - parser.add_argument( - "--mosquitto-host", - default=os.getenv("MOSQUITTO_HOST", "localhost"), - help="Mosquitto host (also use env variable MOSQUITTO_HOST)", - ) - parser.add_argument( - "--mosquitto-port", - type=int, - default=int(os.getenv("MOSQUITTO_PORT", 8883)), - help="Mosquitto port (also use env variable MOSQUITTO_PORT)", - ) - parser.add_argument( - "--mosquitto-timeout", - type=int, - default=int(os.getenv("MOSQUITTO_TIMEOUT", 60)), - help="Mosquitto timeout (also use env variable MOSQUITTO_TIMEOUT)", - ) - parser.add_argument( - "--mosquitto-username", - default=os.getenv("MOSQUITTO_USERNAME", None), - help="Mosquitto username (also use env variable MOSQUITTO_USERNAME)", - ) - parser.add_argument( - "--mosquitto-password", - default=os.getenv("MOSQUITTO_PASSWORD", None), - help="Mosquitto password (also use env variable MOSQUITTO_PASSWORD)", - ) - parser.add_argument( - "--mosquitto-tls", - action="store_true", - default=False, - help="Setup TLS when talking to mosquitto host", - ) - parser.add_argument( - "--mosquitto-transport", - default=os.getenv("MOSQUITTO_TRANSPORT", "tcp"), - choices=["tcp", "websockets"], - help="Mosquitto transport (also use env variable MOSQUITTO_TRANSPORT)", - ) - parser.add_argument( - "--mosquitto-topic-prefix", - default=os.getenv("MOSQUITTO_TOPIC_PREFIX", "perf"), - help="Mosquitto topic prefix (also use env variable MOSQUITTO_TOPIC_PREFIX)", - ) - - -def add_s3_opts(parser): - parser.add_argument( - "--s3-aws-access-key-id", - default=os.getenv("S3_AWS_ACCESS_KEY_ID", "abcdef"), - help="AWS S3 access key ID (also use env variable S3_AWS_ACCESS_KEY_ID)", - ) - parser.add_argument( - "--s3-aws-region", - default=os.getenv("S3_AWS_REGION", "us-east-1"), - help="AWS S3 region (also use env variable S3_AWS_REGION)", - ) - parser.add_argument( - "--s3-aws-secret-access-key", - default=os.getenv("S3_AWS_SECRET_ACCESS_KEY", "abcdef"), - help="AWS S3 secret access key (also use env variable S3_AWS_SECRET_ACCESS_KEY)", - ) - parser.add_argument( - "--s3-bucket", - default=os.getenv("S3_BUCKET", "abcdef"), - help="AWS S3 bucket (also use env variable S3_BUCKET)", - ) - parser.add_argument( - "--s3-endpoint", - default=os.getenv("S3_ENDPOINT", "abcdef"), - help="AWS S3 endpoint (also use env variable S3_ENDPOINT)", - ) - - -def add_locust_opts(parser): - # Is this a simple local runner or master or worker? - parser.add_argument( - "--locust-local-runner", - action="store_true", - default=True if os.getenv("LOCUST_LOCAL_RUNNER", "true") == "true" else False, - help="Make this a local runner (also use env variable LOCUST_LOCAL_RUNNER)", - ) - parser.add_argument( - "--locust-master-runner", - action="store_true", - default=True if os.getenv("LOCUST_MASTER_RUNNER", "false") == "true" else False, - help="Make this a master runner which does not do any requests (also use env variable LOCUST_MASTER_RUNNER)", - ) - parser.add_argument( - "--locust-worker-runner", - action="store_true", - default=True if os.getenv("LOCUST_WORKER_RUNNER", "false") == "true" else False, - help="Make this a worker runner (also use env variable LOCUST_WORKER_RUNNER)", - ) - - # Master specific parameters - parser.add_argument( - "--locust-master-expect-workers", - dest="expect_workers", - type=int, - default=int(os.getenv("LOCUST_MASTER_EXPECT_WORKERS", 1)), - help="How many workers to expect before starting the test (also use env variable LOCUST_MASTER_EXPECT_WORKERS)", - ) - - # Worker specific parameters - parser.add_argument( - "--locust-worker-master-host", - dest="master_host", - default=os.getenv("LOCUST_WORKER_MASTER_HOST", "localhost"), - help="Master host to connect to port 5557 (also use env variable LOCUST_WORKER_MASTER_HOST)", - ) - - # Locust run parameters - parser.add_argument( - "--locust-num-clients", - dest="num_clients", - type=int, - default=int(os.getenv("LOCUST_NUM_CLIENTS", 100)), - help="Locust number of clients (also use env variable LOCUST_NUM_CLIENTS)", - ) - parser.add_argument( - "--locust-hatch-rate", - dest="hatch_rate", - type=float, - default=float(os.getenv("LOCUST_HATCH_RATE", 10)), - help="Locust hatch rate (also use env variable LOCUST_HATCH_RATE)", - ) - parser.add_argument( - "--locust-host", - dest="host", - default=os.getenv("LOCUST_HOST", "http://rbac.qa.svc:8080"), - help="Locust host to test (also use env variable LOCUST_HOST)", - ) - parser.add_argument( - "--locust-stop-timeout", - dest="stop_timeout", - type=int, - default=int(os.getenv("LOCUST_STOP_TIMEOUT", 10)), - help="Locust stop timeout (also use env variable LOCUST_STOP_TIMEOUT)", - ) - parser.add_argument( - "--locust-wait-for-worker-timeout", - dest="worker_wait_timeout", - type=int, - default=int(os.getenv("LOCUST_WAIT_FOR_WORKER_TIMEOUT", 120)), - help="Locust timeout [s] for waiting until worker pods are ready. (also use env variable LOCUST_WAIT_FOR_WORKER_TIMEOUT)", - ) - - # Our test specific parameters - parser.add_argument( - "--test-duration", - type=int, - default=os.getenv("TEST_DURATION", 100), - help="Test duration (also use env variable TEST_DURATION)", - ) - parser.add_argument( - "--test-requests", - type=int, - default=os.getenv("TEST_REQUESTS", 0), - help="Number of requests - if non-0, this overrides test duration (also use env variable TEST_REQUESTS)", - ) - parser.add_argument( - "--test-url-suffix", - default=os.getenv("TEST_URL_SUFFIX", "/api/rbac/v1"), - help="Test host URL suffix (also use env variable TEST_URL_SUFFIX)", - ) - - -def add_tables_def_opts(parser): - parser.add_argument( - "--tables-definition", - type=argparse.FileType("r"), - default=open(os.getenv("TABLES_DEFINITION", "tables.yaml"), "r"), - help="File defining tables and SQL to create them (also use env variable TABLES_DEFINITION)", - ) diff --git a/opl/cluster_read.py b/opl/cluster_read.py deleted file mode 100755 index 73fd30d7..00000000 --- a/opl/cluster_read.py +++ /dev/null @@ -1,675 +0,0 @@ -import logging -import argparse -import csv -import yaml -import json -import subprocess -import re -import requests -import os -import jinja2 -import jinja2.exceptions -import boto3 -import urllib3 -import tempfile - -from . import data -from . import date -from . import status_data -from . import retry - - -def execute(command): - p = subprocess.run( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if p.returncode != 0 or len(p.stderr) != 0: - stderr = p.stderr.decode().strip().replace("\n", "\t") - stdout = p.stdout.decode().strip().replace("\n", "\t") - logging.error( - f"Failed to execute command '{command}' - returned stdout '{stdout}', stderr '{stderr}' and returncode '{p.returncode}'" - ) - result = None - else: - result = p.stdout.decode().strip() - - return result - - -def redact_sensitive_headers(data: dict): - # Lower-case list of sensitive data in header - sensitive_headers = ["authorization", "set-cookie", "x-api-key", "cookie"] - - redacted_headers = {} - for header, value in data.items(): - if header.lower() in sensitive_headers: - redacted_headers[header] = "" - else: - redacted_headers[header] = value - return redacted_headers - - -def _debug_response(r): - """ - Print various info about the requests response. Should be called when - request failed - """ - logging.error("URL = %s" % r.url) - logging.error("Request headers = %s" % redact_sensitive_headers(r.request.headers)) - logging.error("Response headers = %s" % redact_sensitive_headers(r.headers)) - logging.error("Response status code = %s" % r.status_code) - logging.error("Response content = %s" % r.content[:500]) - raise Exception("Request failed") - - -def dir_path(path): - """ - Utility function to be used in Argparse to check for argument is a directory. - """ - if os.path.isdir(path): - return path - else: - raise argparse.ArgumentTypeError(f"{path} is not directory") - - -class BasePlugin: - def __init__(self, args): - self.args = args - - def measure(self, ri, **args): - pass - - def _dump_raw_data(self, name, mydata): - """ - Dumps raw data for monitoring plugins into CSV files (first column - for timestamp, second for value) into provided directory. - """ - if self.args.monitoring_raw_data_dir is None: - return - - file_name = re.sub("[^a-zA-Z0-9-]+", "_", name) + ".csv" - file_path = os.path.join(self.args.monitoring_raw_data_dir, file_name) - - logging.debug(f"Dumping raw data ({len(mydata)} rows) to {file_path}") - with open(file_path, "w", newline="") as csvfile: - csvwriter = csv.writer(csvfile) - csvwriter.writerow(["timestamp", name]) - csvwriter.writerows(mydata) - - @staticmethod - def add_args(parser): - pass - - -class PrometheusMeasurementsPlugin(BasePlugin): - def _get_token(self): - if self.args.prometheus_token is None: - self.args.prometheus_token = execute("oc whoami -t") - if self.args.prometheus_token is None: - raise Exception("Failsed to get token") - return self.args.prometheus_token - - def measure(self, ri, name, monitoring_query, monitoring_step): - logging.debug( - f"/Getting data for {name} using Prometheus query {monitoring_query} and step {monitoring_step}" - ) - - assert ( - ri.start is not None and ri.end is not None - ), "We need timerange to approach Prometheus" - # Get data from Prometheus - url = f"{self.args.prometheus_host}:{self.args.prometheus_port}/api/v1/query_range" - headers = { - "Content-Type": "application/json", - } - if not self.args.prometheus_no_auth: - headers["Authorization"] = f"Bearer {self._get_token()}" - params = { - "query": monitoring_query, - "step": monitoring_step, - "start": ri.start.timestamp(), - "end": ri.end.timestamp(), - } - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - response = requests.get( - url, headers=headers, params=params, verify=False, timeout=60 - ) - if not response.ok or response.headers["Content-Type"] != "application/json": - _debug_response(response) - - # Check that what we got back seems OK - json_response = response.json() - logging.debug("Response: %s" % json_response) - assert json_response["status"] == "success", "'status' needs to be 'success'" - assert "data" in json_response, "'data' needs to be in response" - assert ( - "result" in json_response["data"] - ), "'result' needs to be in response's 'data'" - assert ( - len(json_response["data"]["result"]) != 0 - ), "missing 'response' in response's 'data'" - assert ( - len(json_response["data"]["result"]) == 1 - ), "we need exactly one 'response' in response's 'data'" - assert ( - "values" in json_response["data"]["result"][0] - ), "we need expected form of response" - - mydata = [ - (i[0], float(i[1])) for i in json_response["data"]["result"][0]["values"] - ] - stats = data.data_stats([i[1] for i in mydata]) - self._dump_raw_data(name, mydata) - return name, stats - - @staticmethod - def add_args(parser): - parser.add_argument( - "--prometheus-host", - default="https://prometheus-k8s.openshift-monitoring.svc", - help="Prometheus server to talk to", - ) - parser.add_argument( - "--prometheus-port", - type=int, - default=9091, - help="Port Prometheus is listening on", - ) - parser.add_argument( - "--prometheus-token", - default=None, - help='Authorization token without the "Bearer: " part. If not provided, we will try to get one with "oc whoami -t"', - ) - parser.add_argument( - "--prometheus-no-auth", - action="store_true", - help="Do not send auth headers to Prometheus", - ) - - -class GrafanaMeasurementsPlugin(BasePlugin): - def _sanitize_target(self, target): - target = target.replace("$Node", self.args.grafana_node) - target = target.replace("$Interface", self.args.grafana_interface) - target = target.replace("$Cloud", self.args.grafana_prefix) - return target - - @retry.retry_on_traceback(max_attempts=10, wait_seconds=1) - def measure(self, ri, name, grafana_target, grafana_enritchment={}, grafana_include_vars=False): - assert ( - ri.start is not None and ri.end is not None - ), "We need timerange to approach Grafana" - if ri.start.strftime("%s") == ri.end.strftime("%s"): - return name, None - - # Metadata for the request - headers = { - "Accept": "application/json, text/plain, */*", - } - if self.args.grafana_token is not None: - headers["Authorization"] = "Bearer %s" % self.args.grafana_token - params = { - "target": [self._sanitize_target(grafana_target)], - "from": int(ri.start.timestamp()), - "until": round(ri.end.timestamp()), - "format": "json", - } - url = f"{self.args.grafana_host}:{self.args.grafana_port}/api/datasources/proxy/{self.args.grafana_datasource}/render" - - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - r = requests.post( - url=url, headers=headers, params=params, timeout=60, verify=False - ) - if ( - not r.ok - or r.headers["Content-Type"] != "application/json" - or r.json() == [] - ): - _debug_response(r) - logging.debug("Response: %s" % r.json()) - - points = [float(i[0]) for i in r.json()[0]["datapoints"] if i[0] is not None] - stats = data.data_stats(points) - - # Add user defined data to the computed stats - if grafana_enritchment is not None and grafana_enritchment != {}: - stats["enritchment"] = grafana_enritchment - - # If requested, add info about what variables were used to the computed stats - if grafana_include_vars: - stats["variables"] = { - "$Node": self.args.grafana_node, - "$Interface": self.args.grafana_interface, - "$Cloud": self.args.grafana_prefix, - } - - return name, stats - - @staticmethod - def add_args(parser): - parser.add_argument( - "--grafana-host", default="", help="Grafana server to talk to" - ) - parser.add_argument( - "--grafana-chunk-size", - type=int, - default=10, - help="How many metrices to obtain from Grafana at one request", - ) - parser.add_argument( - "--grafana-port", - type=int, - default=11202, - help="Port Grafana is listening on", - ) - parser.add_argument( - "--grafana-prefix", - default="satellite62", - help="Prefix for data in Graphite", - ) - parser.add_argument( - "--grafana-datasource", type=int, default=1, help="Datasource ID in Grafana" - ) - parser.add_argument( - "--grafana-token", - default=None, - help='Authorization token without the "Bearer: " part', - ) - parser.add_argument( - "--grafana-node", - default="satellite_satperf_local", - help="Monitored host node name in Graphite", - ) - parser.add_argument( - "--grafana-interface", - default="interface-em1", - help="Monitored host network interface name in Graphite", - ) - - -class PerformanceInsightsMeasurementPlugin(BasePlugin): - def get_formatted_metric_query(self, metric_query): - return [{"Metric": metric_query}] - - def measure(self, requested_info, name, identifier, metric_query, metric_step): - logging.debug( - f"/Getting data for {identifier} using PI query {metric_query} with monitoring interval {metric_step}" - ) - - assert ( - requested_info.start is not None and requested_info.end is not None - ), "We need timerange to approach AWS PI service" - - assert ( - self.args.aws_pi_access_key_id is not None - and self.args.aws_pi_secret_access_key is not None - ), "We need AWS access key and secret key to create the client for accessing PI service" - - # Create a low-level service client - aws_session = boto3.session.Session( - aws_access_key_id=self.args.aws_pi_access_key_id, - aws_secret_access_key=self.args.aws_pi_secret_access_key, - region_name=self.args.aws_pi_region_name, - ) - aws_client = aws_session.client("pi") - response = aws_client.get_resource_metrics( - ServiceType="RDS", - Identifier=identifier, - StartTime=requested_info.start, - EndTime=requested_info.end, - MetricQueries=self.get_formatted_metric_query(metric_query), - PeriodInSeconds=metric_step, - ) - - # Check that what we got back seems OK - logging.debug(f"Response: {response}") - assert len(response["MetricList"]) > 0, "'MetricList' should not be empty" - assert ( - response["MetricList"][0]["Key"]["Metric"] == metric_query - ), "'metric_query' needs to be in response" - assert ( - len(response["MetricList"][0]["DataPoints"]) > 0 - ), "'DataPoints' needs to be in response" - - points = [ - data_point["Value"] - for data_point in response["MetricList"][0]["DataPoints"] - if "Value" in data_point - ] - if len(points) < len(response["MetricList"][0]["DataPoints"]): - logging.info( - f"Value is missing in the AWS PI datapoints, total data points: {len(response['MetricList'][0]['DataPoints'])}, available values: {len(points)}" - ) - stats = data.data_stats(points) - return name, stats - - @staticmethod - def add_args(parser): - parser.add_argument( - "--aws-pi-access-key-id", - default=os.getenv("AWS_PI_READ_ONLY_ACCESS_KEY_ID"), - help="The aws access key to use when creating the client for accessing PI service", - ) - parser.add_argument( - "--aws-pi-secret-access-key", - default=os.getenv("AWS_PI_READ_ONLY_SECRET_ACCESS_KEY"), - help="The aws secret key to use when creating the client for accessing PI service", - ) - parser.add_argument( - "--aws-pi-region-name", - default="us-east-1", - help="The name of the aws region associated with the client", - ) - - -class ConstantPlugin(BasePlugin): - def measure(self, ri, name, constant): - """ - Just store given constant - """ - return name, constant - - -class EnvironmentPlugin(BasePlugin): - def measure(self, ri, name, env_variable): - """ - Just get value of given environment variable - """ - return name, os.environ.get(env_variable, None) - - -class CommandPlugin(BasePlugin): - def measure(self, ri, name, command, output="text"): - """ - Execute command "command" and return result as per its "output" configuration - """ - # Execute the command - result = execute(command) - - # Sanitize command response - if result is not None: - if output == "text": - pass - elif output == "json": - result = json.loads(result) - elif output == "yaml": - result = yaml.load(result, Loader=yaml.SafeLoader) - else: - raise Exception(f"Unexpected output type '{output}' for '{name}'") - - return name, result - - -class CountLinePlugin(BasePlugin): - def measure( - self, - ri, - config, - output="text", - ): - """ - Execute command "command" and return result as per its "output" configuration - """ - name = config["name"] - log_source_command = config["log_source_command"] - result = execute(log_source_command).splitlines() - - output = {} - output["all"] = len(result) - - for pattern_name, pattern_value in config.items(): - if not pattern_name.startswith("log_regexp_"): - continue - pattern_key = pattern_name[len("log_regexp_") :] - pattern_regexp = re.compile(pattern_value) - counter = 0 - for line in result: - if pattern_regexp.search(line): - counter += 1 - output[pattern_key] = counter - - return name, output - - -class CopyFromPlugin(BasePlugin): - def measure(self, ri, name, copy_from): - """ - Just return value from previously answered item - """ - if ri.sd is None: - return name, None - else: - return name, ri.sd.get(copy_from) - - -class TestFailMePlugin(BasePlugin): - def measure(self, ri, name, **kwargs): - """ - Just raise an exception. Mean for tests only. - """ - _ = 1 / 0 - - -PLUGINS = { - "test_fail_me": TestFailMePlugin, - "constant": ConstantPlugin, - "env_variable": EnvironmentPlugin, - "command": CommandPlugin, - "copy_from": CopyFromPlugin, - "log_source_command": CountLinePlugin, - "monitoring_query": PrometheusMeasurementsPlugin, - "grafana_target": GrafanaMeasurementsPlugin, - "metric_query": PerformanceInsightsMeasurementPlugin, -} - - -def config_stuff(config): - """ - "config" is yaml loadable stuff - either opened file object, or string - containing yaml formated data. First of all we will run it through - Jinja2 as a template with env variables to expand - """ - - class MyLoader(jinja2.BaseLoader): - """ - Our custom wide open and possibly unsecure jinja2 loader - - Main template is stored as a string, but also capable of loading - templates to satisfy things like: - - {% extends "../something.yaml" %} - - or: - - {% import '../something.yaml' as something %} - - It is very similar to `jinja2.FileSystemLoader('/')` but can also - handle loading files with relative path. - """ - - def __init__(self, main_template): - self.main_template = main_template - - def get_source(self, environment, path): - if path == "main_template": - return self.main_template, None, lambda: True - - if not os.path.exists(path): - raise jinja2.exceptions.TemplateNotFound(path) - - mtime = os.path.getmtime(path) - with open(path) as f: - source = f.read() - - return source, path, lambda: mtime == os.path.getmtime(path) - - if not isinstance(config, str): - config = config.read() - - env = jinja2.Environment(loader=MyLoader(config)) - template = env.get_template("main_template") - - config_rendered = template.render(os.environ) - return yaml.load(config_rendered, Loader=yaml.SafeLoader) - - -class RequestedInfo: - def __init__( - self, config, start=None, end=None, args=argparse.Namespace(), sd=None - ): - """ - "config" is input for config_stuff function - "start" and "end" are datetimes needed if config file contains some - monitoring items to limit from and to time of returned monitoring - data - """ - self.config = config_stuff(config) - self.start = start - self.end = end - self.args = args - self.sd = sd - - self._index = 0 # which config item are we processing? - self._token = None # OCP token - we will take it from `oc whoami -t` if needed - self.measurement_plugins = ( - {} - ) # objects to use for measurements (it's 'measure()' method) by key in config - - # Register plugins - for name, plugin in PLUGINS.items(): - try: - self.register_measurement_plugin(name, plugin(args)) - except Exception as e: - logging.warning(f"Failed to register plugin {name}: {e}") - - def register_measurement_plugin(self, key, instance): - self.measurement_plugins[key] = instance - - def get_config(self): - return self.config - - def __iter__(self): - return self - - def _find_plugin(self, keys): - for key in keys: - if key in self.measurement_plugins: - return self.measurement_plugins[key] - return False - - def __next__(self): - """ - Gives tuple of key and value for every item in the config file - """ - i = self._index - self._index += 1 - if i < len(self.config): - if self._find_plugin(self.config[i].keys()): - instance = self._find_plugin(self.config[i].keys()) - name = list(self.config[i].keys())[1] - try: - if name == "log_source_command": - output = instance.measure(self, self.config[i]) - else: - output = instance.measure(self, **self.config[i]) - except Exception as e: - logging.exception( - f"Failed to measure {self.config[i]['name']}: {e}" - ) - output = (None, None) - return output - else: - raise Exception(f"Unknown config '{self.config[i]}'") - else: - raise StopIteration - - -def doit(args): - if args.requested_info_string: - config = f""" - - name: requested-info-string - command: {args.requested_info_string} - output: {args.requested_info_outputtype} - """ - else: - config = args.requested_info_config - - sd = status_data.StatusData(tempfile.NamedTemporaryFile().name) - - requested_info = RequestedInfo( - config, - args.monitoring_start, - args.monitoring_end, - args=args, - sd=sd, - ) - - if args.render_config: - print(yaml.dump(requested_info.get_config(), width=float("inf"))) - else: - for k, v in requested_info: - print(f"{k}: {v}") - - -def main(): - parser = argparse.ArgumentParser( - description="Run commands defined in a config file and show output", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--requested-info-config", - type=argparse.FileType("r"), - help="File with list of commands to run", - ) - parser.add_argument( - "--requested-info-string", help="Ad-hoc command you want to run" - ) - parser.add_argument( - "--requested-info-outputtype", - default="text", - choices=["text", "json", "yaml"], - help='Ad-hoc command output type, default to "text"', - ) - parser.add_argument( - "--monitoring-start", - type=date.my_fromisoformat, - help="Start of monitoring interval in ISO 8601 format in UTC with seconds precision", - ) - parser.add_argument( - "--monitoring-end", - type=date.my_fromisoformat, - help="End of monitoring interval in ISO 8601 format in UTC with seconds precision", - ) - parser.add_argument( - "--monitoring-raw-data-dir", - type=dir_path, - help="Provide a direcotory if you want raw monitoring data to be dumped in CSV files form", - ) - parser.add_argument( - "--render-config", action="store_true", help="Just render config" - ) - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - for name, plugin in PLUGINS.items(): - plugin.add_args(parser) - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - if args.requested_info_config is None and args.requested_info_string is None: - logging.error( - "At least one of '--requested-info-config' or '--requested-info-string' needs to be set" - ) - return 1 - if ( - args.requested_info_config is not None - and args.requested_info_string is not None - ): - logging.error( - "Only one of '--requested-info-config' or '--requested-info-string' can be set" - ) - return 1 - - logging.debug(f"Args: {args}") - - doit(args) diff --git a/opl/cluster_read_example.yaml b/opl/cluster_read_example.yaml deleted file mode 100644 index 93145e4c..00000000 --- a/opl/cluster_read_example.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- name: parameters.cluster.pods.upload-service.count - command: oc -n qa get pods -l app=upload-service -o name | wc -l -- name: parameters.cluster.pods.insights-puptoo.count - command: oc -n qa get pods -l app=insights-puptoo -o name | wc -l -- name: parameters.cluster.pods.insights-storage-broker.count - command: oc -n qa get pods -l app=insights-storage-broker -o name | wc -l - -# TODO: We should query deployment config here I think -- name: parameters.cluster.pods.vulnerability-engine-listener.resources - command: oc -n qa get pods -l deploymentconfig=vulnerability-engine-listener -o json | python3 -c "import sys, json; print(json.dumps(json.load(sys.stdin)['items'][0]['spec']['containers'][0]['resources']))" - output: json - -- name: measurements.insights-inventory-mq-service.cpu - monitoring_query: sum(pod_name:container_cpu_usage:sum{pod_name=~'insights-inventory-mq-service-.*',namespace='qa'}) - monitoring_step: 60 - -- name: measurements.satellite.load - grafana_target: $Cloud.$Node.load.load.shortterm - -- name: parameters.test_description - constant: This test is about this and that - -- name: parameters.env.JOB_NAME_ID - env_variable: JOB_NAME_ID - -- name: measurements.logs.openshift-pipelines.pipelines-as-code-controller - log_source_command: oc -n openshift-pipelines logs --since=10h --all-containers --selector app.kubernetes.io/component=controller,app.kubernetes.io/instance=default,app.kubernetes.io/name=controller,app.kubernetes.io/part-of=pipelines-as-code,app=pipelines-as-code-controller - log_regexp_error: '"level":"error"' - log_regexp_warning: '"level":"warning"' diff --git a/opl/cluster_read_sat.yaml b/opl/cluster_read_sat.yaml deleted file mode 100644 index 23479c26..00000000 --- a/opl/cluster_read_sat.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Minimal set of values (generally relevant on any host) -- name: measurements.satellite.load.load.shortterm - grafana_target: $Cloud.$Node.load.load.shortterm -- name: measurements.satellite.memory.memory-used - grafana_target: $Cloud.$Node.memory.memory-used -- name: measurements.satellite.swap.swap-used - grafana_target: $Cloud.$Node.swap.swap-used -- name: measurements.satellite.disk_octets.read - grafana_target: scale(sum($Cloud.$Node.*.disk_octets.read), 8) -- name: measurements.satellite.disk_octets.write - grafana_target: scale(sum($Cloud.$Node.*.disk_octets.write), 8) -- name: measurements.satellite.eth1.if_octets.tx - grafana_target: scale($Cloud.$Node.$Interface.if_octets.tx, 8) -- name: measurements.satellite.eth1.if_octets.rx - grafana_target: scale($Cloud.$Node.$Interface.if_octets.rx, 8) -# Extra values specifically for Satellite 6 -- name: measurements.satellite.processes-httpd.ps_rss - grafana_target: $Cloud.$Node.processes-httpd.ps_rss -- name: measurements.satellite.processes-ruby.ps_rss - grafana_target: $Cloud.$Node.processes-ruby.ps_rss -- name: measurements.satellite.processes-dynflow_executor.ps_rss - grafana_target: $Cloud.$Node.processes-dynflow_executor.ps_rss -- name: measurements.satellite.processes-postgres.ps_rss - grafana_target: $Cloud.$Node.processes-postgres.ps_rss -- name: measurements.satellite.processes-Tomcat.ps_rss - grafana_target: $Cloud.$Node.processes-Tomcat.ps_rss -- name: measurements.satellite.processes-qpidd.ps_rss - grafana_target: $Cloud.$Node.processes-qpidd.ps_rss -- name: measurements.satellite.processes-qdrouterd.ps_rss - grafana_target: $Cloud.$Node.processes-qdrouterd.ps_rss -- name: measurements.satellite.processes-httpd.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-httpd.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-ruby.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-ruby.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-dynflow_executor.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-dynflow_executor.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-postgres.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-postgres.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-Tomcat.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-Tomcat.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-qpidd.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-qpidd.ps_cputime.user, 0.0001) -- name: measurements.satellite.processes-qdrouterd.ps_cputime.user - grafana_target: scale($Cloud.$Node.processes-qdrouterd.ps_cputime.user, 0.0001) -- name: measurements.satellite.postgresql-candlepin.pg_n_tup_c-del - grafana_target: $Cloud.$Node.postgresql-candlepin.pg_n_tup_c-del -- name: measurements.satellite.postgresql-candlepin.pg_n_tup_c-ins - grafana_target: $Cloud.$Node.postgresql-candlepin.pg_n_tup_c-ins -- name: measurements.satellite.postgresql-candlepin.pg_n_tup_c-upd - grafana_target: $Cloud.$Node.postgresql-candlepin.pg_n_tup_c-upd -- name: measurements.satellite.postgresql-foreman.pg_n_tup_c-del - grafana_target: $Cloud.$Node.postgresql-foreman.pg_n_tup_c-del -- name: measurements.satellite.postgresql-foreman.pg_n_tup_c-ins - grafana_target: $Cloud.$Node.postgresql-foreman.pg_n_tup_c-ins -- name: measurements.satellite.postgresql-foreman.pg_n_tup_c-upd - grafana_target: $Cloud.$Node.postgresql-foreman.pg_n_tup_c-upd diff --git a/opl/data.py b/opl/data.py deleted file mode 100644 index 2b75ea53..00000000 --- a/opl/data.py +++ /dev/null @@ -1,319 +0,0 @@ -import datetime -import logging -import math -import statistics -import time - - -class WaitForDataAndSave: - def __init__(self, data_db, storage_db, queries, save_here): - self.data_db = data_db - self.storage_db = storage_db - self.queries = queries - self.save_here = save_here - self.batch_size = 100 - self.expected_count = self._get_expected_count() - - def _get_expected_count(self): - cursor = self.storage_db.cursor() - sql = self.queries["get_expected_count"] - cursor.execute(sql) - return int(cursor.fetchone()[0]) - - def _get_remaining_count(self): - cursor = self.storage_db.cursor() - sql = self.queries["get_remaining_count"] - cursor.execute(sql) - return int(cursor.fetchone()[0]) - - def _get_remaining(self, batch_number): - cursor = self.storage_db.cursor() - sql = self.queries["get_remaining"] - cursor.execute(sql, (self.batch_size, batch_number * self.batch_size)) - return tuple([row[0] for row in cursor.fetchall()]) - - def _check_these(self, batch): - count = 0 - logging.debug(f"Checking these: {','.join(batch[:3])}... ({len(batch)} total)") - if len(batch) == 0: - return 0 - data_cursor = self.data_db.cursor() - sql = self.queries["read_these"] - data_cursor.execute(sql, (batch,)) - for row in data_cursor.fetchall(): - logging.debug(f"Saving row {row}") - self.save_here.add(row) - count += 1 - return count - - def wait_common_db_change(self): - """ - It might take some time before finished messages lands in data DB, - so wait for some change (without checking that change we see is for - some system in question). If query returns 0, it is also indication - we are ready to go. - """ - - logging.debug("Waiting for some change in the DB") - count_old = None - data_cursor = self.data_db.cursor() - sql = self.queries["get_all_done_count"] - while True: - data_cursor.execute(sql) - count = data_cursor.fetchone()[0] - - # This is first pass through the loop - if count_old is None: - count_old = count - time.sleep(10) - continue - - # Finally, there was a change - - if count == 0: - logging.info( - f"Finally, count is {count} (was {count_old}), so we can go on" - ) - break - - if count != count_old: - logging.info(f"Finally, count {count_old} changed to {count}") - break - - # Wait some more - logging.debug(f"Count is still only {count}, waiting") - count_old = count - time.sleep(10) - - def process(self): - logging.debug(f"Going to process {self.expected_count} items") - iteration = 0 - iteration_wait = 10 - batch_wait = 0.1 - found_in_total = 0 - found_recently = [ - 1 - ] # track how many items we have found in iterations - seed with fake 1, so we do not exit at first pass - found_recently_size = 100 - while True: - found_in_iteration = 0 - remaining = self._get_remaining_count() - batches_count = int(remaining / self.batch_size) + 1 - logging.debug( - f"Iteration {iteration} running with {batches_count} batches for {remaining} remaining items" - ) - - # Go through all remaining values (from storage DB) in batches - # and attempt to get dates from data DB - for batch_number in range(batches_count): - batch = self._get_remaining(batch_number) - found_count = self._check_these(batch) - logging.debug( - f"In iteration {iteration} batch {batch_number} we have found {found_count} new items" - ) - found_in_iteration += found_count - time.sleep(batch_wait) - - # Detect cases when we have not found any new items for too long - found_recently.append(found_in_iteration) - while len(found_recently) > found_recently_size: - found_recently.pop(0) - if sum(found_recently) == 0: - raise Exception( - f"Nothing found in last {len(found_recently)} iterations, giving up" - ) - - # Are we done? - if remaining == found_in_iteration: - logging.info( - f"We are done in iteration {iteration} with all {self.expected_count} items" - ) - found_in_total += found_in_iteration - break - - iteration += 1 - found_in_total += found_in_iteration - time.sleep(iteration_wait) - - return found_in_total - - -def percentile(data, percent): - if not data: - return None - - data.sort() - k = (len(data) - 1) * percent / 100 - # Python 2.x returns float for floor an ceil, so cast to int - f = int(math.floor(k)) - c = int(math.ceil(k)) - if f == c: - return data[int(k)] - - d0 = data[f] * (c - k) - d1 = data[c] * (k - f) - return d0 + d1 - - -def create_bins(data, precision, bins_number=10): - bins = [] - borders = [] - min_data = min(data) - max_data = max(data) - bin_size = (max_data - min_data) / bins_number - - borders.append(min_data) - for x in range(bins_number): - bins.append((min_data + (bin_size * x), min_data + (bin_size * (x + 1)))) - borders.append(min_data + (bin_size * (x + 1))) - - return bins, borders - - -def find_bin(value, bins): - for i in range(0, len(bins)): - if bins[i][0] <= value < bins[i][1]: - return i - return -1 - - -def histogram(data, precision=1): - if len(data) == 0: - return [0], [0, 1] - - bins, borders = create_bins(data, precision) - counts = [0] * len(bins) - - for value in data: - bin_index = find_bin(value, bins) - counts[bin_index] += 1 - - return counts, borders - - -def data_stats(data): - if len(data) == 0: - return {"samples": 0} - - data_len_before = len(data) - data = [i for i in data if isinstance(i, datetime.datetime) or math.isfinite(i)] - count_strange = data_len_before - len(data) - if count_strange > 0: - logging.warning( - f"There were {count_strange} NaN/Inf values in the data. Filtered them out." - ) - - non_zero_data = [i for i in data if i != 0] - - if isinstance(data[0], int) or isinstance(data[0], float): - q25 = percentile(data, 25) - q75 = percentile(data, 75) - q90 = percentile(data, 90) - q95 = percentile(data, 95) - q99 = percentile(data, 99) - q999 = percentile(data, 99.9) - return { - "samples": len(data), - "min": min(data), - "max": max(data), - "sum": sum(data), - "mean": statistics.mean(data), - "non_zero_mean": ( - statistics.mean(non_zero_data) if len(non_zero_data) > 0 else 0.0 - ), - "median": statistics.median(data), - "non_zero_median": ( - statistics.median(non_zero_data) if len(non_zero_data) > 0 else 0.0 - ), - "stdev": statistics.stdev(data) if len(data) > 1 else 0.0, - "range": max(data) - min(data), - "percentile25": q25, - "percentile75": q75, - "percentile90": q90, - "percentile95": q95, - "percentile99": q99, - "percentile999": q999, - "iqr": q75 - q25, - } - elif isinstance(data[0], datetime.datetime): - return { - "samples": len(data), - "min": min(data), - "max": max(data), - "mean": (max(data) - min(data)) / len(data), - "range": max(data) - min(data), - } - else: - raise Exception(f"Do not know how to get stats for list of {type(data[0])}") - - -def get_hist(data): - hist_counts, hist_borders = histogram(data) - hist_counts = [float(i) for i in hist_counts] - hist_borders = [float(i) for i in hist_borders] - out = [] - for i in range(len(hist_counts)): - out.append(((hist_borders[i], hist_borders[i + 1]), hist_counts[i])) - return out - - -def visualize_hist(data): - for i in get_hist(data): - print(f"<{i[0][0]:.2f}, {i[0][1]:.2f})\t: {i[1]}") - - -def get_rps(data, bucket_size=None, granularity=None): - """ - For given list of timestamps "data", count RPS values "bucket_size" - long interval floating across data (this is set to 10 by default - to stabilize data a bit, but you can set to 1) - """ - if len(data) == 0: - return [] - - out = [] - data_max = max(data) - bucket_start = min(data) - if bucket_size is None: - bucket_size = (data_max - bucket_start) / 30 - bucket_size = max(bucket_size, 10) - if granularity is None: - granularity = bucket_size / 5 - granularity = max(granularity, 1) - bucket_end = bucket_start + bucket_size - logging.debug( - "Counting RPS for %d data points with min %d and max %d with bucket_size=%d and granularity=%d", - len(data), - bucket_start, - data_max, - bucket_size, - granularity, - ) - - while bucket_start <= data_max: - bucket = [i for i in data if bucket_start <= i < bucket_end] - - # If this is last interval, only consider actual length of - # the interval, also check for special case case - if len(bucket) == 1 and bucket[0] == bucket_start: - bucket_duration = 1 - elif bucket_end > data_max: - bucket_duration = max(bucket) - bucket_start + 1 - else: - bucket_duration = bucket_size - - try: - rps = len(bucket) / bucket_duration - except ZeroDivisionError: - logging.warning( - "Empty bucket %s - %s when counting RPS", bucket_start, bucket_end - ) - - out.append(0) - else: - out.append(rps) - - bucket_start += granularity - bucket_end += granularity - - return out diff --git a/opl/date.py b/opl/date.py deleted file mode 100644 index 763b2157..00000000 --- a/opl/date.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import datetime - - -def my_fromisoformat(string): - """ - My limited version of datetime.datetime.fromisoformat() function. Only - accepts one format, only in UTC. Once we are able to run on newer python, - replace this with said function. - """ - if string[-1] == "Z": - string = string[:-1] + "+00:00" - if string[-6] != "+" and string[-5] != "+": - logging.warning(f"Date {string} do not have TZ info, assuming '+00:00'") - string += "+00:00" - if string[-3] != ":": - string = string[:-2] + ":" + string[-2:] - if string.endswith("+00:00"): - string_tz = datetime.timezone.utc - elif string.endswith("+01:00"): - string_tz = datetime.timezone(datetime.timedelta(hours=1)) - elif string.endswith("+02:00"): - string_tz = datetime.timezone(datetime.timedelta(hours=2)) - else: - raise ValueError(f"I do not know how to handle timezone of {string}") - string = string[:-6] - try: - out = datetime.datetime.strptime(string, "%Y-%m-%dT%H:%M:%S.%f") - except ValueError: - out = datetime.datetime.strptime(string, "%Y-%m-%dT%H:%M:%S") - out = out.replace(tzinfo=string_tz) - return out - - -def get_now_str() -> str: - """ - return current datetime in UTC string format - """ - return datetime.datetime.now(datetime.timezone.utc).isoformat() - - -def get_now() -> datetime.datetime: - """ - return current datetime in UTC datetime format - """ - return datetime.datetime.now(datetime.timezone.utc) diff --git a/opl/gen.py b/opl/gen.py deleted file mode 100644 index 1ca0c889..00000000 --- a/opl/gen.py +++ /dev/null @@ -1,85 +0,0 @@ -import random -import datetime -import base64 -import json -import uuid -import string - - -def get_auth_header(account, user, org_id): - data = { - "identity": { - "account_number": account, - "auth_type": "cert-auth", - "org_id": org_id, - "type": "User", - "user": { - "username": user, - "email": user + "@example.com", - "is_org_admin": True, - }, - "internal": { - "org_id": org_id, - }, - "system": { - "cn": None, - }, - } - } - return base64.b64encode(json.dumps(data).encode("UTF-8")) - - -def gen_datetime(plus_hours=None): - utc_now = datetime.datetime.now(tz=datetime.timezone.utc) - if plus_hours is None: - return utc_now.isoformat() - else: - add_hours = datetime.timedelta(hours=plus_hours) - return (utc_now + add_hours).isoformat() - - -def gen_account(): - return str(random.randrange(1000000, 10000000)) - - -def gen_uuid(): - return str(uuid.uuid4()) - - -def gen_subscription_manager_id(): - return gen_uuid() - - -def gen_insights_id(): - return gen_uuid() - - -def gen_machine_id(): - return gen_uuid() - - -def gen_ipv4(): - data = [str(random.randint(1, 255)) for i in range(4)] - return ".".join(data) - - -def gen_ipv6(): - return f"{random.randrange(16**4):x}:{random.randrange(16**4):x}::{random.randrange(16**4):x}:{random.randrange(16**4):x}:{random.randrange(16**4):x}" - - -def gen_mac(): - data = ["%02x" % random.randrange(256) for i in range(6)] - return ":".join(data) - - -def gen_hostname(): - return "".join(random.choices(string.ascii_lowercase, k=25)) + ".example.com" - - -def gen_string(size=10): - return "".join(random.choice(string.printable) for i in range(size)) - - -def gen_safe_string(size=10): - # starting with "u" to specify it is a username - return "u" + "".join(random.choice(string.ascii_lowercase) for _ in range(size - 1)) diff --git a/opl/generators/__init__.py b/opl/generators/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/opl/investigator/README.md b/opl/investigator/README.md deleted file mode 100644 index 89892905..00000000 --- a/opl/investigator/README.md +++ /dev/null @@ -1,169 +0,0 @@ -Results investigator -==================== - -This tool (see `../pass_or_fail.py` in one level up directory) is supposed -to load historical results of given test, compare it with new result for -the same test and decide if new test result is PASS or FAIL. - -You can configure multiple things as of now: - -1. How to get historical results of the test (supports ElasticSearch, CSV - and directory of JSON files) -3. How to load new result (support just JSON file) -4. What method to use to actually find if new result is out of safe bounds - (we mostly use `if new result is biggeer than max or smaller than min - of historical data, it is FAIL`, but it is easy to implement more) -6. What metrics from the JSONs to compare (e.g. `results.rps`, - `monitoring.pod.cpu.mean` and `monitoring.pod.memory.mean`) -8. Optionally you can also configure where to store metadata about - decision the script done. This is useful to keep track about trends - (supports ElasticSearch and CSV) - -See `sample_config.yaml` for example configuration. This is what each -section is for: - -`history:` ----------- - -This specifies from where we should get historical data. Sample config -uses ElasticSearch plugin to retrieve it. Most important part here is -`es_query` which is described in ElasticSearch docs: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - -This query is used to only filter documents that are useful to us: -we only want documents for same test (you can use status data field -`name`) which were running with same parameters and so on. - -That query is Jinja2 template-able, so you can include: - - - term: - parameters.cluster.pods.yupana.count: "{{ current.get('parameters.cluster.pods.yupana.count') }}" - -to dynamically obtain the value from new result and use it to filter for -historical results. - -There are other plugins you can use to retrieve distorical data: - - * `elasticsearch` - Retrieves historical data from ElasticSearch - and is described above - - * `csv` - CSV file with rows being historical results and columns - individual data sets. Example of a CSV file: - - id,name,results.duration - run-2022-01-07T20:57:40+00:00,Test XYZ,152 - run-2022-01-08T03:01:07+00:00,Test XYZ,148 - run-2022-01-08T22:16:30+00:00,Test XYZ,155 - run-2022-01-09T04:20:04+00:00,Test XYZ,151 - run-2022-01-11T01:16:47+00:00,Test XYZ,144 - - * `sd_dir` - directory with status data files from past experiments - which allows filtering by matching various fields before loading data. - Below is example where we load data from SD files whose `name` matches - `name` value from current result. `matchers` is Jinja2 template again: - - type: sd_dir - dir: /tmp/historical_sd_storage/ - matchers: | - name: "{{ current.get('name') }}" - - -`current:` ----------- - -This specifies from where we should load new (`current`) test result -we will be evaluating. - -There is only choice now that loads current result from status data file. - -This can be overwriten by `--current-file` command line option. - - -`methods:` ----------- - -Alows you to specify list of checks you want to use to check results. -These checks are defined in `check.py`. Impractical example: - - methods: - - check_by_stdev_1 - - check_by_stdev_2 - - check_by_stdev_3 - - check_by_trim_stdev_1 - - check_by_trim_stdev_2 - - check_by_error_1 - - check_by_error_2 - - check_by_error_3 - - check_by_error_4 - - check_by_error_5 - - check_by_perc_20 - - check_by_perc_40 - - check_by_perc_60 - - check_by_perc_80 - - check_by_perc_100 - - check_by_min_max_7_1 - - check_by_min_max_7_2 - - check_by_min_max_7_3 - -This is optional and if not present or empty, default set of checks will -be used. - - -`sets:` -------- - -This list status data paths (with *numerical* values) where it makes sense -to evaluate for PASS/FAIL. E.g. you definitely want to include something -like `results.rps` or `measurements.cpu.mean`, but adding -`parameters.test_started.timestamp` might not be useful (because it does -not represent test result comparable across historical runs - this -timestamp is simply always different, based on when the test was running, -so it does not make sense to compare it across historical results). - -It also allws you to define what check methods (other then these defined in -default `methods:` should apply to this metric. You can also provide -additional possitional args if the check method needs them. - -Expected data structure looks like this: - - sets: - - name: metric1 - methods: - - name: check_by_provided_min_max - args: - - 4.5 - - 5.5 - - name: metric2 - -Default check methods from `methods:` is always added and if no method -is specified, script default method is added. - -If this is not a list but a string like in the example below, it is first -rendered via Jinja2 and only then parsed as YAML to get the final list: - - sets: | - {% if current.get('parameters.cli').startswith('experiment/reg-average.py ') %} - - name: results.items.avg_duration - {% else %} - - results.duration - {% endif %} - - name: measurements.satellite.swap.swap-used.mean - - -`decisions:` ------------- - -This is optional and serves to record internal stats about evaluation -process. Every time we do some PASS/FAIL/ERROR decision on any metric -(as defined in `sets`), we record that decision (it's parameters and -result) into system defined here. - -As of now you can use these decisions storage plugins: - - * `elasticsearch` - stores decisions to ElasticSearch index. Then in - Kibana you can have investigation of decision trends dashboards or so. - * `csv` - stores all the decisions for current test in a CSV file - (overwritten every time the tool is invoked) - -This can be turned off with `--dry-run` command line option. diff --git a/opl/investigator/__init__.py b/opl/investigator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/opl/investigator/check.py b/opl/investigator/check.py deleted file mode 100644 index 49a0206d..00000000 --- a/opl/investigator/check.py +++ /dev/null @@ -1,211 +0,0 @@ -import collections -import inspect -import logging -import statistics - - -def _count_deviation(value, lower_boundary, upper_boundary): - if lower_boundary <= value <= upper_boundary: - return None - else: - dist = min(abs(lower_boundary - value), abs(value - upper_boundary)) - try: - frac = dist / abs(upper_boundary - lower_boundary) - except ZeroDivisionError: - frac = 1 - logging.debug( - f"_count_deviation({value}, {lower_boundary}, {upper_boundary}): dist={dist} frac={frac}" - ) - return frac - - -def _calculate_lower_upper_boundary(data, mean, comparator): - """Returns the calculated lower and upper boundary of the data - - Args: - data: collected history data - mean: mean of the collected history data - comparator: defines the type of comparison to be done - - Returns: - tuple with lower and upper boundary - """ - assert isinstance(data, list), "Data provided have to be a list" - lower_boundary = float(mean - (mean - min(data))) - upper_boundary = float(mean + (max(data) - mean)) - if comparator == "lte_max": - return (float("-inf"), upper_boundary) - elif comparator == "gte_min": - return (lower_boundary, float("inf")) - else: - return (lower_boundary, upper_boundary) - - -def _check_by_min_max(data, value, comparator): - """Checks the value range using lower and upper boundary. - If the value is within given range it is a PASS else a FAIL - - Be cautions when using comparator other than None: - - Imagine these historical data measurements.memory_used: - - 100, 105, 95, 102, 98 - - And check configured with comparator being lte_max. - - Now imagine your new value, the value you are checking is 0. - That check would say PASS even though it clearly looks - suspicious. Not only because "memory can not be 0", but more - importantly because "the improvement is so massive, that is - looks suspicious and most probably just point on some error - with collecting that data". - - Args: - data: collected history data - value: value to be checked against - comparator: defines the type of comparison to be done - - Returns: - Boolean value - """ - logging.debug(f"data={data} and value={value}") - mean = statistics.mean(data) - lower_boundary, upper_boundary = _calculate_lower_upper_boundary( - data, mean, comparator - ) - logging.info( - f"value={value}, data len={len(data)} mean={mean:.03f}, i.e. boundaries={lower_boundary:.03f}--{upper_boundary:.03f}" - ) - info = collections.OrderedDict( - [ - ("method", inspect.stack()[1][3]), - ("value", value), - ("data len", len(data)), - ("data mean", mean), - ("data min", float(min(data))), - ("data max", float(max(data))), - ("lower_boundary", lower_boundary), - ("upper_boundary", upper_boundary), - ] - ) - return lower_boundary <= value <= upper_boundary, info - - -def _check_by_stdev(data, value, num_deviations): - logging.debug(f"data={data} and value={value}") - mean = statistics.mean(data) - stdev = statistics.stdev(data) - acceptable_deviation = stdev * num_deviations - lower_boundary = float(mean - acceptable_deviation) - upper_boundary = float(mean + acceptable_deviation) - logging.info( - f"value={value}, data len={len(data)} mean={mean:.03f}, stdev={stdev:.03f}, boundaries={lower_boundary:.03f}--{upper_boundary:.03f}" - ) - info = collections.OrderedDict( - [ - ("method", inspect.stack()[1][3]), - ("value", value), - ("data len", len(data)), - ("data mean", mean), - ("data stdev", stdev), - ("data min", float(min(data))), - ("data max", float(max(data))), - ("lower_boundary", lower_boundary), - ("upper_boundary", upper_boundary), - ] - ) - return lower_boundary <= value <= upper_boundary, info - - -def check_by_iqr(data, value): - """Checks if the current value is within the interquartile range of the previous values""" - logging.debug(f"data={data} and value={value}") - mean = statistics.mean(data) - quantiles = statistics.quantiles(data) - lower_boundary = float(quantiles[0]) - upper_boundary = float(quantiles[2]) - logging.info( - f"value={value}, data len={len(data)} mean={mean:.03f}, boundaries={lower_boundary:.03f}--{upper_boundary:.03f}" - ) - info = collections.OrderedDict( - [ - ("method", inspect.stack()[0][3]), - ("value", value), - ("data len", len(data)), - ("data mean", mean), - ("data quantiles", quantiles), - ("data min", float(min(data))), - ("data max", float(max(data))), - ("lower_boundary", lower_boundary), - ("upper_boundary", upper_boundary), - ] - ) - return lower_boundary <= value <= upper_boundary, info - - -def check_by_min_max_0_1(data, value): - """Checks if the current value is within the min/max range of previous values""" - return _check_by_min_max(data, value, None) - - -def check_by_lte_max(data, value): - """Checks if the current value is less than max range of previous values""" - return _check_by_min_max(data, value, "lte_max") - - -def check_by_gte_min(data, value): - """Checks if the current value is more than min range of previous values""" - return _check_by_min_max(data, value, "gte_min") - - -def check_by_stdev_1(data, value): - """Checks if the current value is within 1 standard deviations of the mean of previous values""" - return _check_by_stdev(data, value, 1) - - -def check_by_stdev_2(data, value): - """Checks if the current value is within 2 standard deviations of the mean of previous values""" - return _check_by_stdev(data, value, 2) - - -def check_by_stdev_3(data, value): - """Checks if the current value is within 2 standard deviations of the mean of previous values""" - return _check_by_stdev(data, value, 3) - -def check_by_provided_min_max(_, value, provided_min, provided_max): - """Checks if the current value is simply withing provided min and max values (historical values not used at all)""" - info = collections.OrderedDict( - [ - ("method", inspect.stack()[0][3]), - ("value", value), - ("lower_boundary", provided_min), - ("upper_boundary", provided_max), - ] - ) - return provided_min <= value <= provided_max, info - - -def check(methods, data, value, description="N/A", verbose=True): - assert value is not None, "Value to check should not be None" - - for method in methods: - assert method["name"] in globals(), f"Check method '{method['name']}' defined" - - results = [] - info_all = [] - for method in methods: - method_name = method["name"] - method_args = method.get("args", []) - result, info = globals()[method_name](data, value, *method_args) - results.append(result) - logging.info(f"{method_name}({', '.join([str(i) for i in method_args])}) value {value} returned {'PASS' if result else 'FAIL'}") - - info_full = collections.OrderedDict() - info_full["description"] = description - info_full["result"] = "PASS" if result else "FAIL" - info_full.update(info) - info_full["deviation"] = _count_deviation( - value, info["lower_boundary"], info["upper_boundary"] - ) - info_all.append(info_full) - return results, info_all diff --git a/opl/investigator/config.py b/opl/investigator/config.py deleted file mode 100644 index e6dfd359..00000000 --- a/opl/investigator/config.py +++ /dev/null @@ -1,172 +0,0 @@ -import logging -import os - -import jinja2 - -import yaml - - -def finalize_sets(args): - """Sets need to have methods configured. - - Before, "sets" was just a list of metrics we want to check. - - Now, "sets" is a datastructure which refferences methods (and their - parameters) that are supposed to be used for checkig. Looks like this: - - [ - { - "name": "metric_that_is_usually_10" - "methods": [ - { - "name": "check_value_from_to", - "args": [8, 12], - }, - ] - }, - ] - - To keep backwards functionality and provide simplified way of - configuration, if set is just list, it is converted to above structore. - - If no check method is provided, default one is added. - - If "methods" was configured, these check methods are added to all "sets". - """ - assert len(args.sets) > 0, "Some sets have to be configured" - - # If this is old format, convert it to new one - if isinstance(args.sets[0], str): - new = [] - for s in args.sets: - new.append({"name": s}) - args.sets = new - - # Make sure all sets have methods list - for s in args.sets: - if "methods" not in s: - s["methods"] = [] - - # If methods were specified, make sure it applies to all sets - if args.methods: - for m in args.methods: - for s in args.sets: - m_present = False - for sm in s["methods"]: - if sm["name"] == m: - m_present = True - break - if not m_present: - s["methods"].append({"name": m}) - - # If no check method configured, use default one - for s in args.sets: - if len(s["methods"]) == 0: - s["methods"].append({"name": "check_by_min_max_0_1"}) - - -def render_sets(args, template_data): - if not isinstance(args.sets, str): - logging.debug("No need to render, sets is not a string") - finalize_sets(args) - return - - logging.debug( - f"Rendering Jinja2 template sets {args.sets} with data {template_data}" - ) - env = jinja2.Environment(loader=jinja2.DictLoader({"sets": args.sets})) - template = env.get_template("sets") - rendered = template.render(template_data) - logging.debug(f"Rendered Jinja2 template sets {rendered}") - args.sets = yaml.load(rendered, Loader=yaml.SafeLoader) - finalize_sets(args) - - -def render_query(args, template_data): - logging.debug( - f"Rendering Jinja2 template query {args.history_es_query} with data {template_data}" - ) - env = jinja2.Environment(loader=jinja2.DictLoader({"query": args.history_es_query})) - template = env.get_template("query") - rendered = template.render(template_data) - logging.debug(f"Rendered Jinja2 template query {rendered}") - args.history_es_query = yaml.load(rendered, Loader=yaml.SafeLoader) - - -def render_matchers(args, template_data): - logging.debug( - f"Rendering Jinja2 template matchers {args.history_matchers} with data {template_data}" - ) - env = jinja2.Environment( - loader=jinja2.DictLoader({"matchers": args.history_matchers}) - ) - template = env.get_template("matchers") - rendered = template.render(template_data) - logging.debug(f"Rendered Jinja2 template matchers {rendered}") - args.history_matchers = yaml.load(rendered, Loader=yaml.SafeLoader) - - -def load_config_finish(args, sd): - template_data = {"current": sd, "environ": os.environ} - render_sets(args, template_data) - if args.history_type == "elasticsearch": - render_query(args, template_data) - if args.history_type == "sd_dir": - render_matchers(args, template_data) - - -def load_config(conf, fp): - """ - Load config from yaml file pointer and add to conf which is an ArgParser namespace - """ - data = yaml.load(fp, Loader=yaml.SafeLoader) - logging.debug(f"Loaded config from {fp.name}: {data}") - - conf.history_type = data["history"]["type"] - conf.current_type = data["current"]["type"] - conf.methods = data["methods"] if "methods" in data else [] - conf.sets = data["sets"] - conf.decisions_type = data["decisions"]["type"] - - if conf.history_type == "csv": - conf.history_file = open(data["history"]["file"], "r") - - if conf.history_type == "elasticsearch": - conf.history_es_server = data["history"]["es_server"] - assert not conf.history_es_server.endswith("/") - conf.history_es_index = data["history"]["es_index"] - conf.history_es_query = data["history"]["es_query"] - if "es_server_user" in data["history"]: - conf.history_es_server_user = data["history"]["es_server_user"] - conf.history_es_server_pass_env_var = data["history"][ - "es_server_pass_env_var" - ] - if "es_server_verify" in data["history"]: - conf.history_es_server_verify = data["history"]["es_server_verify"] - else: - conf.history_es_server_verify = True - - if conf.history_type == "sd_dir": - conf.history_dir = data["history"]["dir"] - conf.history_matchers = data["history"]["matchers"] - - if conf.current_file is None: - if conf.current_type == "status_data": - conf.current_file = open(data["current"]["file"], "r") - - if conf.decisions_type == "elasticsearch": - conf.decisions_es_server = data["decisions"]["es_server"] - assert not conf.decisions_es_server.endswith("/") - conf.decisions_es_index = data["decisions"]["es_index"] - if "es_server_user" in data["decisions"]: - conf.decisions_es_server_user = data["decisions"]["es_server_user"] - conf.decisions_es_server_pass_env_var = data["decisions"][ - "es_server_pass_env_var" - ] - if "es_server_verify" in data["decisions"]: - conf.decisions_es_server_verify = data["decisions"]["es_server_verify"] - else: - conf.decisions_es_server_verify = True - - if conf.decisions_type == "csv": - conf.decisions_filename = data["decisions"].get("file", data["decisions"].get("filename")) diff --git a/opl/investigator/csv_decisions.py b/opl/investigator/csv_decisions.py deleted file mode 100644 index 77019dd5..00000000 --- a/opl/investigator/csv_decisions.py +++ /dev/null @@ -1,28 +0,0 @@ -import csv -import datetime -import os - - -def store(filename, decisions): - # This is our workaround on how to add additional metadata about the decision - job_name = os.environ.get("JOB_NAME", "") - build_url = os.environ.get("BUILD_URL", "") - - for decision in decisions: - decision["job_name"] = job_name - decision["build_url"] = build_url - decision["uploaded"] = datetime.datetime.now(tz=datetime.timezone.utc).isoformat() - - fieldnames = [] - for d in decisions: - for k in d.keys(): - if k not in fieldnames: - fieldnames.append(k) - - with open(filename, "w") as fp: - writer = csv.DictWriter(fp, fieldnames=fieldnames) - - writer.writeheader() - - for decision in decisions: - writer.writerow(decision) diff --git a/opl/investigator/csv_loader.py b/opl/investigator/csv_loader.py deleted file mode 100644 index 367ed2c8..00000000 --- a/opl/investigator/csv_loader.py +++ /dev/null @@ -1,21 +0,0 @@ -import csv -import logging - - -def load(fp, columns): - out = {} - - for col in columns: - out[col] = [] - - reader = csv.DictReader(fp) - - for row in reader: - for col in out.keys(): - out[col].append(float(row[col])) - - logging.info( - f"Loaded file {fp.name} and parsed {len(out.keys())} columns with {len(next(iter(out.values())))} rows" - ) - - return out diff --git a/opl/investigator/elasticsearch_decisions.py b/opl/investigator/elasticsearch_decisions.py deleted file mode 100644 index a8d3e570..00000000 --- a/opl/investigator/elasticsearch_decisions.py +++ /dev/null @@ -1,44 +0,0 @@ -import datetime -import json -import logging -import os - -import requests - - -def store(server, index, decisions, **kwargs): - es_server_user = kwargs.get("es_server_user") - decisions_es_server_pass_env_var = kwargs.get("es_server_pass_env_var") - # This is our workaround on how to add additional metadata about the decision - job_name = os.environ.get("JOB_NAME", "") - build_url = os.environ.get("BUILD_URL", "") - - url = f"{server}/{index}/_doc" - headers = { - "Content-Type": "application/json", - } - for decision in decisions: - decision["job_name"] = job_name - decision["build_url"] = build_url - decision["uploaded"] = datetime.datetime.utcnow().isoformat() - - # for k, v in decision.items(): - # print(f">>> {k} = {v} ({type(v)})") - logging.info( - f"Storing decision to ES url={url}, headers={headers} and json={json.dumps(decision)}" - ) - - if es_server_user and decisions_es_server_pass_env_var: - # fetch the password from Jenkins credentials - open_search_password = os.environ.get(decisions_es_server_pass_env_var) - response = requests.post( - url, - auth=requests.auth.HTTPBasicAuth(es_server_user, open_search_password), - headers=headers, - json=decision, - ) - else: - response = requests.post(url, headers=headers, json=decision) - - if not response.ok: - logging.warning(f"Failed to store decision to ES: {response.text}") diff --git a/opl/investigator/elasticsearch_loader.py b/opl/investigator/elasticsearch_loader.py deleted file mode 100644 index f2b6d603..00000000 --- a/opl/investigator/elasticsearch_loader.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -import logging -import tempfile - -import os -import opl.http -import opl.status_data -from requests.auth import HTTPBasicAuth - - -def load(server, index, query, paths, **kwargs): - es_server_user = kwargs.get("es_server_user") - es_server_pass_env_var = kwargs.get("es_server_pass_env_var") - - out = {} - - for path in paths: - out[path] = [] - - url = f"{server}/{index}/_search" - headers = { - "Content-Type": "application/json", - } - data = query - logging.info( - f"Querying ES with url={url}, headers={headers} and json={json.dumps(data)}" - ) - - if es_server_user and es_server_pass_env_var: - # fetch the password from Jenkins credentials - open_search_password = os.environ.get(es_server_pass_env_var) - response = opl.http.get( - url, - auth=HTTPBasicAuth(es_server_user, open_search_password), - headers=headers, - json=data, - ) - else: - response = opl.http.get(url, headers=headers, json=data) - - for item in response["hits"]["hits"]: - logging.debug( - f"Loading data from document ID {item['_id']} with field id={item['_source']['id'] if 'id' in item['_source'] else None} or parameters.run={item['_source']['parameters']['run'] if 'run' in item['_source']['parameters'] else None}" - ) - tmpfile = tempfile.NamedTemporaryFile(prefix=item["_id"], delete=False).name - sd = opl.status_data.StatusData(tmpfile, data=item["_source"]) - for path in paths: - tmp = sd.get(path) - if tmp is not None: - out[path].append(tmp) - - logging.debug(f"Loaded {out}") - return out diff --git a/opl/investigator/sample_config.yaml b/opl/investigator/sample_config.yaml deleted file mode 100644 index 7c0ede90..00000000 --- a/opl/investigator/sample_config.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Get previous test results from here -history: - # type: csv - # file: /tmp/data.csv - - # type: sd_dir - # dir: /tmp/historical_sd_storage/ - # matchers: | - # name: "{{ current.get('name') }}" - - type: elasticsearch - es_server: http://elasticsearch.example.com:9286 - es_index: my-index - es_query: | - query: - bool: - filter: - - term: - name.keyword: "something" - sort: - started: - order: asc - size: 30 - - -# Get results of test run to be compared to historical results -current: - type: status_data - file: /tmp/status-data.json - -# columns in case of CSV input -# JSON paths in case of status_data input -sets: - - name: collumn1 - methods: - - name: check_by_provided_min_max - args: - - 10 - - 20 - - name: collumn2 - -# These check methods will be added to all sets -methods: - - check_by_min_max_0_1 - # - check_by_iqr - # - check_by_stdev_1 - -# Shuld we log our decisions somewhere? -decisions: - # type: csv - # filename: /tmp/decisions.csv - - type: elasticsearch - es_server: http://elasticsearch.example.com:9286 - es_index: my_aa_decisions diff --git a/opl/investigator/sd_dir_loader.py b/opl/investigator/sd_dir_loader.py deleted file mode 100644 index 5e17e41c..00000000 --- a/opl/investigator/sd_dir_loader.py +++ /dev/null @@ -1,37 +0,0 @@ -import logging -import os - -import opl.status_data - - -def _matches(sd, matchers): - for key, val in matchers.items(): - if sd.get(key) != val: - logging.debug(f"File {sd} key {key} does not match {val} != {sd.get(key)}") - return False - - logging.debug(f"File {sd} matches with matchers {matchers}") - return True - - -def load(data_dir, data_matchers, paths): - out = {} - - for path in paths: - out[path] = [] - - for dirpath, dirnames, filenames in os.walk(data_dir): - for f in filenames: - if not f.endswith(".json") or not os.path.isfile(os.path.join(dirpath, f)): - continue - - sd = opl.status_data.StatusData(os.path.join(dirpath, f)) - - if _matches(sd, data_matchers): - for path in paths: - tmp = sd.get(path) - if tmp is not None: - out[path].append(tmp) - - logging.debug(f"Loaded {out}") - return out diff --git a/opl/investigator/status_data_loader.py b/opl/investigator/status_data_loader.py deleted file mode 100644 index 7cdd60cd..00000000 --- a/opl/investigator/status_data_loader.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging - -import opl.status_data - - -def load(fp): - return opl.status_data.StatusData(fp.name) - - -def load_data(sd, paths): - out = {} - - for path in paths: - out[path] = sd.get(path) - if out[path] is None: - logging.warning(f"While loading {sd}, got None for {path}") - - logging.info(f"Loaded file {sd} and parsed {len(out.keys())} paths from it") - - return out diff --git a/opl/junit_cli.py b/opl/junit_cli.py deleted file mode 100755 index 570555d5..00000000 --- a/opl/junit_cli.py +++ /dev/null @@ -1,528 +0,0 @@ -import argparse -import datetime -import json -import logging -import os -import unicodedata -import junitparser -import urllib3 -import re - -import requests - -from . import date - - -def now(): - return datetime.datetime.now(tz=datetime.timezone.utc) - - -class TestCaseWithProp(junitparser.TestCase): - def properties(self): - """ - Iterates through all properties. - """ - props = self.child(junitparser.Properties) - if props is None: - return - for prop in props: - yield prop - - def add_property(self, name, value): - """ - Adds a property to the testsuite. - """ - props = self.child(junitparser.Properties) - if props is None: - props = junitparser.Properties() - self.append(props) - prop = junitparser.Property(name, value) - props.add_property(prop) - - def get_property(self, name, default=None): - """ - Get a property from the testcase - """ - for prop in self.properties(): - if prop.name == name: - return prop.value - return default - - -class JUnitXmlPlus(junitparser.JUnitXml): - @classmethod - def fromfile_or_new(cls, filename): - if os.path.exists(filename): - instance = cls.fromfile(filename) - else: - instance = cls() - instance.filepath = filename - return instance - - def _remove_control_characters(self, s): - return "".join( - ch for ch in s if unicodedata.category(ch)[0] != "C" or ch == "\n" - ) - - def trim_string_fn(self, data, trim_length): - matches = list(re.finditer(r"\S+", data)) - if len(matches) <= trim_length: - return data - # Get the start index of the Nth-to-last word - start_index = matches[-trim_length].start() - return data[start_index:] - - def add_to_suite(self, suite_name, new): - case = TestCaseWithProp(new["name"]) - - suite_found = False - for suite in self: - if suite.name == suite_name: - logging.debug(f"Suite {suite_name} found, going to add into it") - suite_found = True - if not suite_found: - logging.debug(f"Suite {suite_name} not found, creating new one") - suite = junitparser.TestSuite(suite_name) - self.add_testsuite(suite) - - if new["result"] == "PASS": - case.result = [] - elif new["result"] == "FAIL": - case.result = [junitparser.Failure(new["message"])] - elif new["result"] == "ERROR": - case.result = [junitparser.Error(new["message"])] - else: - raise Exception(f"Invalid result {new['result']}") - - case.system_out = "" - if new["system-out"]: - for f in new["system-out"]: - try: - case.system_out += self._remove_control_characters(f.read()) - case.system_out += "\n" - except ValueError as e: - logging.error(f"Failed to load {new['system-out'].name} file: {e}") - case.system_err = "" - if new["system-err"]: - for f in new["system-err"]: - try: - case.system_err += self._remove_control_characters(f.read()) - case.system_err += "\n" - except ValueError as e: - logging.error(f"Failed to load {new['system-err'].name} file: {e}") - - case.system_out = self.trim_string_fn(case.system_out, 1000) - duration = (new["end"] - new["start"]).total_seconds() - case.time = duration - - case.add_property("start", new["start"].isoformat()) - case.add_property("end", new["end"].isoformat()) - - suite.add_testcase(case) - - self.write() - - def get_info(self): - out = [] - for suite in self: - print(f"suite: {suite}") - for prop in suite.properties(): - print(f" property: {prop}") - for case in suite: - case = TestCaseWithProp.fromelem(case) - print(f" case: {case} {case.result}") - for prop in case.properties(): - print(f" property: {prop}") - return "\n".join(out) - - def get_result(self): - RESULTS = ["PASSED", "SKIPPED", "FAILED", "ERROR"] - result = 0 - for suite in self: - for case in suite: - case = TestCaseWithProp.fromelem(case) - if len(case.result) == 0: - pass # result "PASS" can not make overall result worse - elif len(case.result) == 1: - r = case.result[0] - if isinstance(r, junitparser.junitparser.Error): - result = max(result, 3) - elif isinstance(r, junitparser.junitparser.Failure): - result = max(result, 2) - elif isinstance(r, junitparser.junitparser.Skipped): - result = max(result, 1) - else: - raise Exception( - f"No idea how to handle this result type: {r} - {type(r)}" - ) - else: - raise Exception( - f"No idea how to handle this case result: {case.result}" - ) - return RESULTS[result] - - def delete(self): - os.remove(self.filepath) - - def ibutsu_upload(self, host, token, project, verify, file, metadata): - if not verify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - headers = {"Authorization": f"Bearer {token}"} - if metadata: - metadata = json.dumps(self.parse_ibutsu_metadata(metadata)) - res = requests.post( - f"{host}/api/import", - headers=headers, - files={ - "importFile": (file, open(file, "rb"), "text/xml"), - }, - verify=verify, - data={"project": project, "metadata": metadata}, - ) - if not res.ok: - raise Exception(res.text) - else: - logging.debug(res.text) - - def parse_ibutsu_metadata(self, metadata_list): - """Parse the metadata from a set of strings to a dictionary""" - metadata = {} - # Loop through the list of metadata values - for pair in metadata_list: - # Split the key part from the value - key_path, value = pair.split("=", 1) - # Split the key up if it is a dotted path - keys = key_path.split(".") - current_data = metadata - # Loop through all but the last key and create the dictionary structure - for key in keys[:-1]: - if key not in current_data: - current_data[key] = {} - current_data = current_data[key] - # Finally, set the actual value - key = keys[-1] - current_data[key] = value - return metadata - - def upload(self, host, verify, project, token, launch, properties): - def req(method, url, data): - logging.debug(f"Going to do {method} request to {url} with {data}") - response = method(url, json=data, headers=headers, verify=verify) - if not response.ok: - logging.error(f"Request failed: {response.text}") - response.raise_for_status() - logging.debug(f"Request returned {response.json()}") - return response.json() - - def times(ts): - return str(int(ts.timestamp() * 1000)) - - if not verify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Determine launch start and end by taking min from test case - # starts and max from testcase ends - start = None - end = None - suites_times = {} - for suite in self: - suite_start = None - suite_end = None - for case in suite: - case = TestCaseWithProp.fromelem(case) - for prop in case.properties(): - if prop.name == "start": - tmp = date.my_fromisoformat(prop.value) - if start is None or start > tmp: - start = tmp - if suite_start is None or suite_start > tmp: - suite_start = tmp - if prop.name == "end": - tmp = date.my_fromisoformat(prop.value) - if end is None or end < tmp: - end = tmp - if suite_end is None or suite_end < tmp: - suite_end = tmp - suite_start = suite_start if suite_start is not None else now() - suite_end = suite_end if suite_end is not None else now() - suites_times[suite.name] = [suite_start, suite_end] - start = start if start is not None else now() - end = end if end is not None else now() - - # Start a session - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {token}", - } - session = requests.Session() - - # Start launch - url = f"https://{host}/api/v1/{project}/launch" - data = { - "name": launch, - "startTime": times(start), - "mode": "DEFAULT", - "attributes": [], - } - for prop in properties: - name, value = prop.split("=") - data["attributes"].append( - { - "key": name, - "value": value, - } - ) - response = req(session.post, url, data) - launch_id = response["id"] - - # Process all the suites - for suite in self: - suite_start, suite_end = suites_times[suite.name] - - # Start root(suite) item - url = f"https://{host}/api/v1/{project}/item" - data = { - "name": suite.name, - "startTime": times(suite_start), - "type": "suite", - "launchUuid": launch_id, - "attributes": [], - } - for prop in suite.properties(): - data["attributes"].append( - { - "key": prop.name, - "value": prop.value, - } - ) - response = req(session.post, url, data) - suite_id = response["id"] - - # Process all the testcases in the suite - for case in suite: - case = TestCaseWithProp.fromelem(case) - case_start = date.my_fromisoformat(case.get_property("start", now())) - case_end = date.my_fromisoformat(case.get_property("end", now())) - - # Determine case status - if len(case.result) == 0: - result = "passed" - issue = None - elif isinstance(case.result[0], junitparser.junitparser.Error): - result = "failed" - issue = "si001" - elif isinstance(case.result[0], junitparser.junitparser.Failure): - result = "failed" - issue = "ti001" - elif isinstance(case.result[0], junitparser.junitparser.Skipped): - result = "skipped" - issue = "ti001" - else: - raise Exception(f"Unknown result for {case}: {case.result}") - - # Start child(container) item - url = f"https://{host}/api/v1/{project}/item/{suite_id}" - data = { - "name": case.name, - "startTime": times(case_start), - "type": "test", - "launchUuid": launch_id, - "attributes": [], - } - for prop in case.properties(): - data["attributes"].append( - { - "key": prop.name, - "value": prop.value, - } - ) - response = req(session.post, url, data) - case_id = response["id"] - - # Finish parent(container) item - url = f"https://{host}/api/v1/{project}/item/{case_id}" - data = { - "endTime": times(case_end), - "launchUuid": launch_id, - "status": result, - } - if issue is not None: - data["issue"] = {"issueType": issue} - response = req(session.put, url, data) - - # Add log message - url = f"https://{host}/api/v1/{project}/log" - data = { - "launchUuid": launch_id, - "itemUuid": case_id, - "time": times(case_end), - "message": case.system_out, - "level": "info", - } - response = req(session.post, url, data) - if case.system_err: - data["message"] = case.system_err - data["level"] = "error" - response = req(session.post, url, data) - - # Finish root(suite) item - url = f"https://{host}/api/v1/{project}/item/{suite_id}" - data = { - "endTime": times(suite_end), - "launchUuid": launch_id, - } - response = req(session.put, url, data) - - # Finish launch - url = f"https://{host}/api/v1/{project}/launch/{launch_id}/finish" - data = { - "endTime": times(end), - } - response = req(session.put, url, data) - - # Show where we have uploaded data - print(f"Created launch https://{host}/ui/#{project}/launches/all/{launch_id}") - - -def main(): - parser = argparse.ArgumentParser( - description="Manipulate jUnit file", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--file", - default=os.getenv("JUNIT_FILE", "junit.xml"), - help="jUnit file to work with (also use env variable JUNIT_FILE)", - ) - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - subparsers = parser.add_subparsers(dest="action", help="Select one of sub-commands") - - # Create the parser for the "print" command - subparsers.add_parser("print", help="Print content of the file") # noqa: F841 - - # Create the parser for the "result" command - subparsers.add_parser( - "result", help="Print overall result from the file" # noqa: F841 - ) - - # create the parser for the "add" command - parser_add = subparsers.add_parser("add", help="Add testcase into the file") - parser_add.add_argument("--name", required=True, help="Name of the testcase") - parser_add.add_argument( - "--result", - required=True, - choices=["PASS", "FAIL", "ERROR"], - help="Result of the testcase", - ) - parser_add.add_argument( - "--suite", required=True, help="Testsuite this testcase should be in" - ) - parser_add.add_argument( - "--out", - type=argparse.FileType("r"), - nargs="*", - default=[], - help="File with stdout of the testcase", - ) - parser_add.add_argument( - "--err", - type=argparse.FileType("r"), - nargs="*", - default=[], - help="File with stderr of the testcase", - ) - parser_add.add_argument("--message", help="Message when result is failure or error") - parser_add.add_argument( - "--start", - required=True, - type=date.my_fromisoformat, - help="Testcase start time in ISO 8601 format", - ) - parser_add.add_argument( - "--end", - required=True, - type=date.my_fromisoformat, - help="Testcase end time in ISO 8601 format", - ) - - # create the parser for the "ibutsu" command - parser_ibutsu = subparsers.add_parser( - "ibutsu-import", help="Import the file to Ibutsu" - ) - parser_ibutsu.add_argument("--host", required=True, help="Ibutsu host") - parser_ibutsu.add_argument("--token", required=True, help="Ibutsu token") - parser_ibutsu.add_argument("--project", required=True, help="Ibutsu project") - parser_ibutsu.add_argument( - "--noverify", - action="store_true", - help="When talking to Ibutsu ignore certificate verification failures", - ) - parser_ibutsu.add_argument( - "--metadata", - action="append", - help="Additional metadata to set when uploading, in the format of dotted.key.path=value", - ) - - # create the parser for the "upload" command - parser_add = subparsers.add_parser("upload", help="Upload the file to ReportPortal") - parser_add.add_argument("--host", required=True, help="ReportPortal host") - parser_add.add_argument( - "--noverify", - action="store_true", - help="When talking to ReportPortal ignore certificate verification failures", - ) - parser_add.add_argument("--project", required=True, help="ReportPortal project") - parser_add.add_argument("--token", required=True, help="ReportPortal token") - parser_add.add_argument( - "--launch", required=True, help="ReportPortal launch name to use when creating" - ) - parser_add.add_argument( - "--properties", - nargs="*", - default=[], - help="Launch property pairs in a name=value form, space separated, that will be added as ReportPortal attributes", - ) - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - logging.debug(f"Args: {args}") - - junit = JUnitXmlPlus.fromfile_or_new(args.file) - - if args.action == "print": - print(junit.get_info()) - elif args.action == "result": - print(junit.get_result()) - elif args.action == "add": - new = { - "name": args.name, - "result": args.result, - "system-out": args.out, - "system-err": args.err, - "message": args.message, - "start": args.start, - "end": args.end, - } - junit.add_to_suite(args.suite, new) - elif args.action == "upload": - junit.upload( - args.host, - not args.noverify, - args.project, - args.token, - args.launch, - args.properties, - ) - elif args.action == "ibutsu-import": - junit.ibutsu_upload( - args.host, - args.token, - args.project, - not args.noverify, - args.file, - args.metadata, - ) - else: - raise Exception("I do not know what to do") diff --git a/opl/pass_or_fail.py b/opl/pass_or_fail.py deleted file mode 100755 index 31c2813a..00000000 --- a/opl/pass_or_fail.py +++ /dev/null @@ -1,249 +0,0 @@ -import argparse -import collections -import logging -import sys - -import opl.investigator.check -import opl.investigator.config -import opl.investigator.csv_decisions -import opl.investigator.csv_loader -import opl.investigator.elasticsearch_decisions -import opl.investigator.elasticsearch_loader -import opl.investigator.sd_dir_loader -import opl.investigator.status_data_loader - -import tabulate - - -STATUSES = { - 0: "PASS", - 1: "FAIL", - 2: "ERROR", -} - - -def get_stats(checks, key): - per_key = {} - for i in checks: - if i[key] not in per_key: - per_key[i[key]] = { - "PASS": 0, - "FAIL": 0, - "ERROR": 0, - } - per_key[i[key]][i["result"]] += 1 - - # Count score - for k, v in per_key.items(): - score = v["FAIL"] + 10 * v["ERROR"] if "ERROR" in v else 0 - v["SCORE"] = score - - # Reorder data structure for tabulate - per_key_tabulate = [] - for k, v in per_key.items(): - v["name"] = k - per_key_tabulate.append(v) - - # Sort by score - per_key_tabulate = sorted(per_key_tabulate, key=lambda i: i["SCORE"]) - - print( - f"\nStats by {key}:\n\n", - tabulate.tabulate( - per_key_tabulate, headers="keys", tablefmt="simple", floatfmt=".3f" - ), - ) - - -def doit(args): - opl.investigator.config.load_config(args, args.config) - - # Load current data - if args.current_type == "status_data": - current_sd = opl.investigator.status_data_loader.load(args.current_file) - else: - raise Exception("Not supported data source type for current data") - - # Render what needs to be rendered to finish config loading - opl.investigator.config.load_config_finish(args, current_sd) - - sets_list = [s['name'] for s in args.sets] - - # Load data items from current data - current = opl.investigator.status_data_loader.load_data(current_sd, sets_list) - - total = len([v for v in current.values() if v is not None and v != ""]) - if total == 0: - raise Exception( - f"No data available in current result (tried to load {', '.join(sets_list)} but nothing)!" - ) - - # Load historical data - if args.history_type == "csv": - history = opl.investigator.csv_loader.load(args.history_file, sets_list) - elif args.history_type == "elasticsearch": - if ( - hasattr(args, "history_es_server_verify") - and not args.history_es_server_verify - ): - # SSL verification is disabled by default - opl.http.insecure() - history = opl.investigator.elasticsearch_loader.load( - args.history_es_server, - args.history_es_index, - args.history_es_query, - sets_list, - es_server_user=getattr(args, "history_es_server_user", None), - es_server_pass_env_var=getattr( - args, "history_es_server_pass_env_var", None - ), - ) - - elif args.history_type == "sd_dir": - history = opl.investigator.sd_dir_loader.load( - args.history_dir, - args.history_matchers, - sets_list, - ) - else: - raise Exception("Not supported data source type for historical data") - - total = sum([len(v) for v in history.values()]) - if total == 0: - logging.info(f"Current result metrics: {current}") - logging.fatal("No data available in historical results!") - sys.exit(1) - - # Compute if current data matches historical data safe margins - exit_code = 0 - summary = [] - info_all = [] - for s in args.sets: - var = s["name"] - methods = s["methods"] - try: - results, info = opl.investigator.check.check( - methods, history[var], current[var], description=var - ) - except Exception as e: - logging.exception(f"Check on {var} failed with: {e}") - info_all.append({"result": "ERROR", "exception": str(e)}) - summary_this = collections.OrderedDict( - [("data set", var), ("exception", str(e))] - ) - exit_code = 2 - else: - info_all += info - result_overall = False not in results - summary_this = collections.OrderedDict() - summary_this["data set"] = var - summary_this.update({i["method"]: i["result"] for i in info}) - if exit_code == 0 and not result_overall: - exit_code = 1 - - summary.append(summary_this) - - if args.detailed_decisions: - # Only take headers common to all the rows - info_headers = list(info_all[0].keys()) - for i in info_all: - info_headers = [k for k in info_headers if k in i.keys()] - else: - info_headers = [ - "description", - "result", - "method", - "value", - "lower_boundary", - "upper_boundary", - "deviation", - ] - for i in info_all: - for k in list(i.keys()): - if k not in info_headers: - del i[k] - - info_headers_tabulate = dict( - zip(info_headers, info_headers) - ) # https://bitbucket.org/astanin/python-tabulate/issues/39/valueerror-headers-for-a-list-of-dicts-is - - print( - "\n", - tabulate.tabulate( - info_all, headers=info_headers_tabulate, tablefmt="simple", floatfmt=".3f" - ), - ) - print("\n", tabulate.tabulate(summary, headers="keys", tablefmt="simple")) - print(f"\nOverall status: {STATUSES[exit_code]}") - - if args.stats: - get_stats(info_all, "description") - get_stats(info_all, "method") - - if not args.dry_run: - if args.decisions_type == "elasticsearch": - if hasattr(args, "es_server_verify") and not args.es_server_verify: - # disable SSL verification - opl.http.insecure() - opl.investigator.elasticsearch_decisions.store( - args.decisions_es_server, - args.decisions_es_index, - info_all, - es_server_user=getattr(args, "decisions_es_server_user", None), - es_server_pass_env_var=getattr( - args, "decisions_es_server_pass_env_var", None - ), - ) - if args.decisions_type == "csv": - opl.investigator.csv_decisions.store(args.decisions_filename, info_all) - - if not args.dry_run: - if exit_code == 0: - result = "PASS" - elif exit_code == 1: - result = "FAIL" - else: - result = "ERROR" - logging.info(f"In {current_sd} setting result to {result}") - current_sd.set("result", result) - current_sd.save() - - return exit_code - - -def main(): - parser = argparse.ArgumentParser( - description="Given historical numerical data, determine if latest result is PASS or FAIL", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--config", - type=argparse.FileType("r"), - required=True, - help="Config file to use", - ) - parser.add_argument( - "--current-file", - type=argparse.FileType("r"), - help="Status data file with results to investigate. Overwrites current.file value from config file", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Investigate result, but do not upload decisions. Meant for debugging", - ) - parser.add_argument( - "--detailed-decisions", - action="store_true", - help="When showing decisions, show all the details", - ) - parser.add_argument("--stats", action="store_true", help="Show statistics") - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - logging.debug(f"Args: {args}") - - sys.exit(doit(args)) diff --git a/opl/retry.py b/opl/retry.py deleted file mode 100644 index 00c747dc..00000000 --- a/opl/retry.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -import time -from functools import wraps - - -def retry_on_traceback(max_attempts=10, wait_seconds=1): - """ - Retries a function until it succeeds or the maximum number of attempts - or wait time is reached. - - This is to mimic `@retry` decorator from Tenacity so we do not depend - on it. - - Args: - max_attempts: The maximum number of attempts to retry the function. - wait_seconds: The number of seconds to wait between retries. - - Returns: - A decorator that retries the wrapped function. - """ - assert max_attempts >= 0, "It does not make sense to have less than 0 retries" - assert wait_seconds >= 0, "It does not make sense to wait less than 0 seconds" - - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - attempt = 0 - while True: - try: - return func(*args, **kwargs) - except Exception as e: - if attempt >= max_attempts: - raise # Reraise the exception after all retries are exhausted - - attempt += 1 - logging.debug( - f"Retrying in {wait_seconds} seconds. Attempt {attempt}/{max_attempts} failed with: {e}" - ) - time.sleep(wait_seconds) - - return wrapper - - return decorator diff --git a/opl/rp_updater.py b/opl/rp_updater.py deleted file mode 100755 index 004da188..00000000 --- a/opl/rp_updater.py +++ /dev/null @@ -1,160 +0,0 @@ -import argparse -import logging - -import opl.http -import opl.status_data - -import tabulate - - -def _get_all_items(args): - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - url = f"https://{args.rp_host}/api/v1/{args.rp_project}/item" - params = { - "filter.eq.launchId": args.rp_launch_id, - "page.size": 100, - "page.page": 1, - # 'page.sort': 'endTime,desc', - } - - items = [] - while True: - response = opl.http.get( - url, params=params, headers=headers, verify=not args.rp_noverify - ) - items += response["content"] - if len(response["content"]) == 0: - logging.debug(f"Page {params['page.page']} was last page of results") - break - params["page.page"] += 1 - logging.debug(f"Going to query for page {params['page.page']} of results") - - return items - - -def doit_list_tests(args): - items = _get_all_items(args) - - # Add suite name to tests - suites = {} - for i in items: - if i["type"] == "SUITE": - suites[i["id"]] = i - for i in items: - if i["type"] == "TEST": - i["parent_name"] = suites[i["parent"]]["name"] - - table = [ - ( - i["id"], - i["type"], - i["parent_name"] + "/" + i["name"], - i["status"], - i["statistics"]["defects"], - ) - for i in items - if i["type"] == "TEST" - ] - print(tabulate.tabulate(table, headers=("ID", "type", "name", "status", "defects"))) - - -def doit_change_defects(args): - changes = 0 - items = _get_all_items(args) - - defect_group = args.from_defect.split("/")[0] - defect_id = args.from_defect.split("/")[1] - defect_from = {defect_group: {"total": 1, defect_id: 1}} - - defect_to_id = args.to_defect.split("/")[1] - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - url = f"https://{args.rp_host}/api/v1/{args.rp_project}/item" - - for i in items: - if i["type"] != "TEST": - continue - - if i["statistics"]["defects"] != defect_from: - continue - - data = { - "issues": [ - { - "testItemId": i["id"], - "issue": { - "issueType": defect_to_id, - "comment": args.to_defect_comment, - }, - } - ] - } - - opl.http.put(url, headers=headers, data=data, verify=not args.rp_noverify) - changes += 1 - - print(f"Changed {changes} tests from {defect_id} to {defect_to_id}") - - -def main(): - parser = argparse.ArgumentParser( - description="Investigate and modify status data documents in ElasticSearch", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("--rp-host", help="ReportPortal host") - parser.add_argument( - "--rp-noverify", - action="store_true", - help="When talking to ReportPortal ignore certificate verification failures", - ) - parser.add_argument("--rp-project", help="ReportPortal project") - parser.add_argument("--rp-token", help="ReportPortal token") - parser.add_argument("--rp-launch-id", help="ReportPortal launch ID") - - parser.add_argument( - "--dry-run", - action="store_true", - help="Do not actually change data, meant for debugging", - ) - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - - subparsers = parser.add_subparsers(dest="action", help="Select one of sub-commands") - - # create the parser for the "list_tests" command - subparsers.add_parser("list_tests", help="Print tests in given launch") - - # create the parser for the "change_defects" command - parser_change_defects = subparsers.add_parser( - "change_defects", help="Change defects in given launch" - ) - parser_change_defects.add_argument( - "--from-defect", help="Take these defects (e.g. to_investigate/ti001" - ) - parser_change_defects.add_argument( - "--to-defect", help="And change them to these defects (e.g. no_issue/ni001)" - ) - parser_change_defects.add_argument( - "--to-defect-comment", help="Comment to add to the test with changed defect" - ) - - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - opl.http.disable_insecure_request_warnings(args.rp_noverify) - - logging.debug(f"Args: {args}") - - if args.action == "list_tests": - return doit_list_tests(args) - elif args.action == "change_defects": - return doit_change_defects(args) - else: - raise Exception(f"Unknown action '{args.action}'") diff --git a/opl/shovel.py b/opl/shovel.py deleted file mode 100755 index 00e654f6..00000000 --- a/opl/shovel.py +++ /dev/null @@ -1,1179 +0,0 @@ -#!/usr/bin/env python - -import argparse -import datetime -import logging -import requests -import json -import os -import re -import urllib3 -import urllib.parse - -from opl import skelet -from opl import retry - - -@retry.retry_on_traceback(max_attempts=10, wait_seconds=10) -def _requests_get_with_retry(*args, **kwargs): - return requests.get(*args, **kwargs) - - -def _check_response(logger, response): - """Check if requests response is OKish and if not, log useful data and raise exception.""" - try: - response.raise_for_status() - except Exception: - logger.error(f"Request failed with text: {response.text}") - raise - - -def _floor_datetime(obj): - """Floor datetime object to whole second.""" - return obj.replace(microsecond=0) - - -def _ceil_datetime(obj): - """Ceil datetime object to whole second.""" - if obj.microsecond > 0: - obj += datetime.timedelta(seconds=1) - return obj.replace(microsecond=0) - - -def _get_field_value(field, data): - """Return content of filed like foo.bar or .baz or so.""" - if field.startswith("."): - field = field[1:] - - value = None - for f in field.split("."): - if f == "": - continue # skip empty field created e.g. by leading dot - if value is None: - value = data[f] - else: - value = value[f] - - return value - - -def _set_field_value(field, value, data): - """Find field (in doted notation) in data (being changed in place) and set it to value.""" - if field.startswith("@"): - field = field[1:] - - for f in field.split(".")[:-1]: - if f not in data: - data[f] = {} - data = data[f] - - data[field.split(".")[-1]] = value - - -def _figure_out_option(option, data): - """Normalize option value for cases when it can come from data file. Checks for None.""" - if option.startswith("@"): - field = option[1:] - value = _get_field_value(field, data) - if value is None: - raise Exception(f"Can not load {field} in {option}") - else: - return value - else: - if option is None: - raise Exception("Some option was not provided") - else: - return option - - -class pluginBase: - def __init__(self): - self.logger = logging.getLogger(str(self.__class__)) - - def set_args(parser, subparsers): - pass - - -class pluginProw(pluginBase): - def list(self, args): - response = requests.get(f"{args.base_url}/{args.job_name}") - _check_response(self.logger, response) - - # Extract 19-digit numbers using regular expression - numbers = re.findall(r"\b[0-9]{19}\b", response.text) - - # Sort the numbers in numerical order and get the last 10 unique numbers - sorted_numbers = sorted(set(numbers), key=lambda x: int(x)) - last_10_numbers = sorted_numbers[-10:] - for n in last_10_numbers: - print(n) - - def download(self, args): - if os.path.isfile(args.output_path): - raise Exception( - f"File {args.output_path} already present, refusing to overwrite it" - ) - - from_url = f"{args.base_url}/{args.job_name}/{args.job_run_id}/artifacts/{args.run_name}/{args.artifact_path}" - logging.info(f"Downloading {from_url} to {args.output_path}") - response = requests.get(from_url) - _check_response(self.logger, response) - response_content = response.content - - if args.record_link is not None: - try: - data = response.json() - except requests.exceptions.JSONDecodeError: - self.logger.error( - "Failed to parse JSON, ignoring --record-link option" - ) - else: - _set_field_value(args.record_link, from_url, data) - response_content = str.encode( - json.dumps(data, sort_keys=True, indent=4) - ) - - with open(args.output_path, "wb") as f: - f.write(response_content) - - def set_args(self, parser, subparsers): - # Generic Prow options - parser.add_argument( - "--base-url", - default="https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/logs/", - help="Base URL of Prow exporter", - ) - parser.add_argument( - "--job-name", - default="periodic-ci-konflux-ci-e2e-tests-main-load-test-ci-daily-100u", - help="Job name as available in ci-operator/jobs/...", - ) - - # Options for listing Prow runs - parser_list = subparsers.add_parser( - "list", help="List runs for specific Prow job" - ) - parser_list.set_defaults(func=self.list) - - # Options for downloading artifacts from Prow - parser_download = subparsers.add_parser( - "download", help="Download file from Prow run artifacts" - ) - parser_download.set_defaults(func=self.download) - parser_download.add_argument( - "--job-run-id", - required=True, - help="Long run number identifier", - ) - parser_download.add_argument( - "--run-name", - default="load-test-ci-daily-100u", - help="Test name as configured in ci-operator/config/...", - ) - parser_download.add_argument( - "--artifact-path", - default="redhat-appstudio-load-test/artifacts/load-test.json", - help="Path to the artifact in artifacts/ directory", - ) - parser_download.add_argument( - "--output-path", - required=True, - help="Filename where to put downloaded artifact", - ) - parser_download.add_argument( - "--record-link", - help="Optional path in the downloaded JSON where to put download link", - ) - - -class pluginOpenSearch(pluginBase): - def upload(self, args): - self.logger.info(f"Loading document {args.input_file}") - with open(args.input_file, "r") as fp: - values = json.load(fp) - - if args.matcher_field.startswith("."): - args.matcher_field = args.matcher_field[1:] - self.logger.info(f"Looking for field {args.matcher_field}") - matcher_value = _get_field_value(args.matcher_field, values) - if matcher_value is None: - raise Exception( - f"Failed to load {args.matcher_field} from {args.input_file}" - ) - - self.logger.info( - f"Checking if document {args.matcher_field}={matcher_value} is already present" - ) - query = {"query": {"match": {f"{args.matcher_field}": matcher_value}}} - headers = {"Content-Type": "application/json"} - - current_doc_in_es = requests.post( - f"{args.base_url}/{args.index}/_search", - headers=headers, - json=query, - ) - _check_response(self.logger, current_doc_in_es) - current_doc_in_es = current_doc_in_es.json() - - if current_doc_in_es["hits"]["total"]["value"] > 0: - print( - f"Document {args.matcher_field}={matcher_value} is already present, skipping upload" - ) - return - - self.logger.info("Uploading document") - response = requests.post( - f"{args.base_url}/{args.index}/_doc", - headers=headers, - json=values, - ) - _check_response(self.logger, response) - print(f"Uploaded: {response.content}") - - def set_args(self, parser, subparsers): - parser.add_argument( - "--base-url", - default="http://elasticsearch.intlab.perf-infra.lab.eng.rdu2.redhat.com", - help="Base URL of OpenSearch server", - ) - parser.add_argument( - "--index", - default="rhtap_ci_status_data", - help="Index to talk to", - ) - - # Options for uploading document to OpenSearch - parser_upload = subparsers.add_parser( - "upload", help="Upload document to OpenSearch" - ) - parser_upload.set_defaults(func=self.upload) - parser_upload.add_argument( - "--input-file", - required=True, - help="JSON file to upload", - ) - parser_upload.add_argument( - "--matcher-field", - help="Document field which holds unique value identifying the document. Will be used for checking if data exists on the server, value will be taken from input file.", - ) - - -class pluginHorreum(pluginBase): - def _setup(self, args): - # FIXME - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - self.headers = { - "Content-Type": "application/json", - "X-Horreum-API-Key": args.api_token, - } - - if "test_name" in args: - self.logger.debug(f"Getting test id for {args.test_name}") - response = _requests_get_with_retry( - f"{args.base_url}/api/test/byName/{args.test_name}", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - self.test_id = response.json()["id"] - - def upload(self, args): - self.logger.debug(f"Loading file {args.input_file}") - with open(args.input_file, "r") as fd: - self.input_file = json.load(fd) - - self.logger.info("Preparing all the options") - args.test_name = _figure_out_option(args.test_name, self.input_file) - args.start = datetime.datetime.fromisoformat( - _figure_out_option(args.start, self.input_file) - ) - args.end = datetime.datetime.fromisoformat( - _figure_out_option(args.end, self.input_file) - ) - - self._setup(args) - - self.logger.info(f"Looking for field {args.matcher_field}") - matcher_value = _get_field_value(args.matcher_field, self.input_file) - if matcher_value is None: - raise Exception( - f"Failed to load {args.matcher_field} from {args.input_file}" - ) - - self.logger.debug( - f"Searching if result {args.matcher_label}={matcher_value} is already there" - ) - filter_data = {args.matcher_label: matcher_value} - response = _requests_get_with_retry( - f"{args.base_url}/api/dataset/list/byTest/{self.test_id}", - headers=self.headers, - params={"filter": json.dumps(filter_data)}, - verify=False, - ) - _check_response(self.logger, response) - datasets = response.json().get("datasets", []) - if len(datasets) > 0: - print( - f"Result {args.matcher_label}={matcher_value} is already there, skipping upload" - ) - return - - if args.trashed: - self.logger.debug( - "WORKAROUND: Searching if result already there amongst trashed runs" - ) - params = { - "trashed": True, - "limit": args.trashed_workaround_count, - "page": 1, - "sort": "start", - "direction": "Descending", - } - response = _requests_get_with_retry( - f"{args.base_url}/api/run/list/{self.test_id}", - headers=self.headers, - params=params, - verify=False, - ) - _check_response(self.logger, response) - runs = response.json().get("runs", []) - - for run in runs: - # Un-trashed runs were examined already, so we can skipp these - if run["trashed"] is False: - continue - - response = _requests_get_with_retry( - f"{args.base_url}/api/run/{run['id']}/data", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - run_data = response.json() - try: - marker = _get_field_value(args.matcher_field, run_data) - except KeyError: - pass # If matcher was not found in data, we can assume this is not a duplicate - else: - if marker == matcher_value: - print( - f"Result {args.matcher_field}={matcher_value} is trashed, but already there, skipping upload" - ) - return - - logging.info("Uploading") - params = { - "test": args.test_name, - "start": args.start.strftime("%Y-%m-%dT%H:%M:%SZ"), - "stop": args.end.strftime("%Y-%m-%dT%H:%M:%SZ"), - "owner": args.owner, - "access": args.access, - } - response = requests.post( - f"{args.base_url}/api/run/data", - params=params, - headers=self.headers, - data=json.dumps(self.input_file), - verify=False, - ) - _check_response(self.logger, response) - - print(f"Uploaded {args.input_file}: {response.content}") - - def result(self, args): - self.logger.debug(f"Loading file {args.output_file}") - with open(args.output_file, "r") as fd: - self.output_file = json.load(fd) - - self.logger.info("Preparing all the options") - args.test_name = _figure_out_option(args.test_name, self.output_file) - args.start = datetime.datetime.fromisoformat( - _figure_out_option(args.start, self.output_file) - ).astimezone(tz=datetime.timezone.utc) - args.end = datetime.datetime.fromisoformat( - _figure_out_option(args.end, self.output_file) - ).astimezone(tz=datetime.timezone.utc) - - self._setup(args) - - self.logger.debug(f"Loading list of alerting variables for test {self.test_id}") - response = _requests_get_with_retry( - f"{args.base_url}/api/alerting/variables", - params={"test": self.test_id}, - verify=False, - ) - _check_response(self.logger, response) - alerting_variables = response.json() - - start = _floor_datetime(args.start) - end = _ceil_datetime(args.end) - start_str = start.strftime("%Y-%m-%dT%H:%M:%SZ") - end_str = end.strftime("%Y-%m-%dT%H:%M:%SZ") - - change_detected = False - for alerting_variable in alerting_variables: - self.logger.debug( - f"Getting changes for alerting variable {alerting_variable}" - ) - - range_data = { - "range": { - "from": start_str, - "to": end_str, - "oneBeforeAndAfter": True, - }, - "annotation": {"query": alerting_variable["id"]}, - } - response = requests.post( - f"{args.base_url}/api/changes/annotations", - headers=self.headers, - json=range_data, - verify=False, - ) - _check_response(self.logger, response) - response = response.json() - - # Check if the result is not an empty list - if len(response) > 0: - self.logger.info( - f"For {alerting_variable['name']} detected change {response}" - ) - change_detected = True - break - else: - self.logger.info(f"For {alerting_variable['name']} all looks good") - - result = "FAIL" if change_detected else "PASS" - - if args.output_file is None: - print(f"Result is {result}") - return - - self.output_file["result"] = result - - print(f"Writing result to {args.output_file}: {self.output_file['result']}") - with open(args.output_file, "w") as fd: - json.dump(self.output_file, fd, sort_keys=True, indent=4) - - def list(self, args): - self._setup(args) - - params = { - "trashed": False, - "limit": 100, - "page": 1, - } - - self.logger.debug(f"Listing results for test {args.test_name}") - while True: - response = _requests_get_with_retry( - f"{args.base_url}/api/run/list/{self.test_id}", - headers=self.headers, - params=params, - verify=False, - ) - _check_response(self.logger, response) - - runs = response.json().get("runs", []) - if len(runs) == 0: - break - - for i in runs: - print(i["id"]) - - params["page"] += 1 - - def get(self, args): - self._setup(args) - - self.logger.debug(f"Geting data for run {args.run_id}") - response = _requests_get_with_retry( - f"{args.base_url}/api/run/{args.run_id}/data", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - data = response.json() - print(json.dumps(data)) - - def _schema_uri_to_id(self, base_url, schema_uri): - self.logger.debug(f"Geting schema ID for URI {schema_uri}") - response = _requests_get_with_retry( - f"{base_url}/api/schema/idByUri/{schema_uri}", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - schema_id = int(response.json()) - self.logger.debug(f"Schema ID for URI {schema_uri} is {schema_id}") - return schema_id - - def _sanitize_string(self, input_string): - return re.sub(r"[^a-zA-Z0-9]", "_", input_string) - - def _schema_id_labels(self, args, schema_id): - self.logger.debug(f"Getting list of labels for schema ID {schema_id}") - response = _requests_get_with_retry( - f"{args.base_url}/api/schema/{schema_id}/labels", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - data = response.json() - self.logger.debug(f"Obtained {len(data)} labels for schema ID {schema_id}") - - return data - - def schema_label_list(self, args): - self._setup(args) - - schema_id = self._schema_uri_to_id(args.base_url, args.schema_uri) - - labels = self._schema_id_labels(args, schema_id) - - for label in labels: - print(f"{label['id']}\t{label['name']}\t{label['extractors'][0]['jsonpath']}") - - def schema_label_add(self, args): - self._setup(args) - - schema_id = self._schema_uri_to_id(args.base_url, args.schema_uri) - - new_name = ( - self._sanitize_string(args.extractor_jsonpath) - if args.name is None - else args.name - ) - new_ext_name = ( - self._sanitize_string(args.extractor_jsonpath) - if args.extractor_name is None - else args.extractor_name - ) - new = { - "access": args.access, - "owner": args.owner, - "name": new_name, - "extractors": [ - { - "name": new_ext_name, - "jsonpath": args.extractor_jsonpath, - "isarray": args.extractor_isarray, - }, - ], - "function": args.function, - "filtering": args.filtering, - "metrics": args.metrics, - "schemaId": schema_id, - } - - self.logger.debug(f"Adding label to schema id {schema_id}: {new}") - response = requests.post( - f"{args.base_url}/api/schema/{schema_id}/labels", - headers=self.headers, - verify=False, - json=new, - ) - _check_response(self.logger, response) - label_id = int(response.json()) - self.logger.debug(f"Created label ID {label_id} in schema ID {schema_id}") - - def schema_label_update(self, args): - self._setup(args) - - schema_id = self._schema_uri_to_id(args.base_url, args.schema_uri) - - new_name = ( - self._sanitize_string(args.extractor_jsonpath) - if args.name is None - else args.name - ) - new_ext_name = ( - self._sanitize_string(args.extractor_jsonpath) - if args.extractor_name is None - else args.extractor_name - ) - new = { - "access": args.access, - "owner": args.owner, - "name": new_name, - "extractors": [ - { - "name": new_ext_name, - "jsonpath": args.extractor_jsonpath, - "isarray": args.extractor_isarray, - }, - ], - "function": args.function, - "filtering": args.filtering, - "metrics": args.metrics, - "schemaId": schema_id, - } - - # Account options that allow updating existing label - if args.update_by_id is not None or args.update_by_name is not None: - labels = self._schema_id_labels(args, schema_id) - - if args.update_by_id is not None: - label = next((item for item in labels if item["id"] == args.update_by_id), False) - - if label: - # Label with provided id found, lets update it - new["id"] = args.update_by_id - else: - # Label not found - raise KeyError(f"Failed to find label with id {args.update_by_id}") - - elif args.update_by_name is not None: - label = next((item for item in labels if item["name"] == new_name), False) - - if label: - # Label with provided name found, lets update it - new["id"] = label["id"] - else: - # Label not found, shall we fail now? - if args.add_if_missing: - self.logger.warning(f"Label with name {new_name} not found, so adding new one with new ID") - return self.schema_label_add(args) - else: - raise KeyError(f"Failed to find label with name {new_name}") - else: - raise Exception("Either --update-by-id or --update-by-name have to be used") - - if new == label: - self.logger.info(f"Proposed and current label {label['id']} in schema ID {schema_id} are same, nothing to change") - else: - self.logger.debug(f"Updating label in schema id {schema_id}: {new}") - response = requests.put( - f"{args.base_url}/api/schema/{schema_id}/labels", - headers=self.headers, - verify=False, - json=new, - ) - _check_response(self.logger, response) - label_id = int(response.json()) - self.logger.info(f"Updated label ID {label_id} in schema ID {schema_id}") - - def schema_label_delete(self, args): - self._setup(args) - - schema_id = self._schema_uri_to_id(args.base_url, args.schema_uri) - - self.logger.debug(f"Deleting label ID {args.id} from schema ID {schema_id}") - response = requests.delete( - f"{args.base_url}/api/schema/{schema_id}/labels/{args.id}", - headers=self.headers, - verify=False, - ) - _check_response(self.logger, response) - self.logger.debug(f"Deleted label ID {args.id} in schema ID {schema_id}") - - def set_args(self, parser, subparsers): - parser.add_argument( - "--base-url", - default="https://horreum.corp.redhat.com", - help="Base URL of Horreum server", - ) - parser.add_argument( - "--api-token", - required=True, - help="Horreum API token", - ) - - # Options for uploading document to Horreum - subparser = subparsers.add_parser( - "upload", help="Upload file to Horreum if it is not there already" - ) - subparser.set_defaults(func=self.upload) - subparser.add_argument( - "--test-name", - default="load-tests-result", - help="Test name as configured in Horreum, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - subparser.add_argument( - "--input-file", - required=True, - help="JSON file to upload", - ) - subparser.add_argument( - "--matcher-field", - default="runid", - help="JSON file field which holds unique value identifying the document. Will be used for checking if data exists on the server, value will be taken from input file.", - ) - subparser.add_argument( - "--matcher-label", - default=".runid", - help="Label name in Horreum with unique value we use to detect if document is already in Horreum", - ) - subparser.add_argument( - "--owner", - default="rhtap-perf-test-team", - help="Who should be owner of uploaded file in Horreum", - ) - subparser.add_argument( - "--access", - default="PUBLIC", - help="What should be access setting of uploaded file in Horreum", - ) - subparser.add_argument( - "--start", - help="When the test whose JSON file we are uploading started, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - subparser.add_argument( - "--end", - help="When the test whose JSON file we are uploading ended, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - subparser.add_argument( - "--trashed", - action="store_true", - help="Use this if you want to check for presence of a result even amongst trashed runs", - ) - subparser.add_argument( - "--trashed-workaround-count", - default=10, - help="When listing runs (including trashed ones) sorted in descending order, only check this many", - ) - - # Options for detecting no-/change signal - subparser = subparsers.add_parser( - "result", help="Get Horreum no-/change signal for a given time range" - ) - subparser.set_defaults(func=self.result) - subparser.add_argument( - "--test-name", - default="load-tests-result", - help="Test name as configured in Horreum, if prefixed with '@' sign, it is a field name from output file where to load this", - ) - subparser.add_argument( - "--start", - help="Start of the interval for detecting change (ISO 8601 format)", - ) - subparser.add_argument( - "--end", - help="End of the interval for detecting change (ISO 8601 format)", - ) - subparser.add_argument( - "--output-file", - help="If specified, put result into .result of this JSON file", - ) - - # Options for listing results - subparser = subparsers.add_parser( - "list", help="List test run IDs for a test" - ) - subparser.set_defaults(func=self.list) - subparser.add_argument( - "--test-name", - default="load-tests-result", - help="Test name as configured in Horreum", - ) - - # Options for getting full result - subparser = subparsers.add_parser("get", help="Get data for test run ID") - subparser.set_defaults(func=self.get) - subparser.add_argument( - "--run-id", - type=int, - required=True, - help="Test run ID", - ) - - # Options for listing schema labels - subparser = subparsers.add_parser( - "schema-label-list", help="List schema labels" - ) - subparser.set_defaults(func=self.schema_label_list) - subparser.add_argument( - "--schema-uri", - type=str, - required=True, - help="Schema URI identifier (e.g. 'uri:my-schema:0.1')", - ) - - # Options for adding schema label - subparser = subparsers.add_parser( - "schema-label-add", help="Add schema label" - ) - subparser.set_defaults(func=self.schema_label_add) - subparser.add_argument( - "--schema-uri", - type=str, - required=True, - help="Schema URI identifier (e.g. 'uri:my-schema:0.1')", - ) - subparser.add_argument( - "--name", - type=str, - help="Label name. Keep empty and it will de derived from JSON path.", - ) - subparser.add_argument( - "--extractor-name", - type=str, - help="Extractor name. Keep empty and it will de derived from JSON path. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--extractor-jsonpath", - type=str, - required=True, - help="Extractor JSON path expression. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--extractor-isarray", - type=bool, - default=False, - help="If extractor refferencing an array? Defaults to false. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--function", - type=str, - default="", - help="Combination function for the label. Defaults to empty one.", - ) - subparser.add_argument( - "--filtering", - action="store_true", - help="Is label a filtering label? Defaults to false.", - ) - subparser.add_argument( - "--metrics", - action="store_true", - help="Is label a metrics label? Defaults to false.", - ) - subparser.add_argument( - "--access", - choices=["PUBLIC", "PROTECTED", "PRIVATE"], - default="PUBLIC", - help="Access rights for the test. Defaults to 'PUBLIC'.", - ) - subparser.add_argument( - "--owner", - type=str, - required=True, - help="Name of the team that owns the test.", - ) - - # Options for updating schema label - subparser = subparsers.add_parser( - "schema-label-update", help="Update schema label" - ) - subparser.set_defaults(func=self.schema_label_update) - subparser.add_argument( - "--schema-uri", - type=str, - required=True, - help="Schema URI identifier (e.g. 'uri:my-schema:0.1')", - ) - subparser.add_argument( - "--update-by-id", - type=int, - help="Label ID of label you want to update. Only set if you want to update existing label.", - ) - subparser.add_argument( - "--update-by-name", - action="store_true", - help="Label name of label you want to update. Only set if you want to update existing label.", - ) - subparser.add_argument( - "--add-if-missing", - action="store_true", - help="If updating (e.g. by name) and the target label is missing, attempt to create it.", - ) - subparser.add_argument( - "--name", - type=str, - help="Label name. Keep empty and it will de derived from JSON path.", - ) - subparser.add_argument( - "--extractor-name", - type=str, - help="Extractor name. Keep empty and it will de derived from JSON path. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--extractor-jsonpath", - type=str, - required=True, - help="Extractor JSON path expression. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--extractor-isarray", - type=bool, - default=False, - help="If extractor refferencing an array? Defaults to false. Only one extractor allowed now, support for more is TODO.", - ) - subparser.add_argument( - "--function", - type=str, - default="", - help="Combination function for the label. Defaults to empty one.", - ) - subparser.add_argument( - "--filtering", - action="store_true", - help="Is label a filtering label? Defaults to false.", - ) - subparser.add_argument( - "--metrics", - action="store_true", - help="Is label a metrics label? Defaults to false.", - ) - subparser.add_argument( - "--access", - choices=["PUBLIC", "PROTECTED", "PRIVATE"], - default="PUBLIC", - help="Access rights for the test. Defaults to 'PUBLIC'.", - ) - subparser.add_argument( - "--owner", - type=str, - required=True, - help="Name of the team that owns the test.", - ) - - # Options for deleting schema label - subparser = subparsers.add_parser( - "schema-label-delete", help="Delete schema label" - ) - subparser.set_defaults(func=self.schema_label_delete) - subparser.add_argument( - "--schema-uri", - type=str, - required=True, - help="Schema URI identifier (e.g. 'uri:my-schema:0.1')", - ) - subparser.add_argument( - "--id", - type=int, - required=True, - help="Label ID of label you want to delete", - ) - - -class pluginResultsDashboard(pluginBase): - def upload(self, args): - self.input_file = None - if args.input_file is not None: - self.logger.info(f"Loading input file {args.input_file}") - with open(args.input_file, "r") as fd: - self.input_file = json.load(fd) - - self.logger.info("Preparing all the options") - args.date = datetime.datetime.fromisoformat( - _figure_out_option(args.date, self.input_file) - ) - args.group = _figure_out_option(args.group, self.input_file) - args.link = _figure_out_option(args.link, self.input_file) - args.product = _figure_out_option(args.product, self.input_file) - args.release = _figure_out_option(args.release, self.input_file) - args.result = _figure_out_option(args.result, self.input_file) - args.result_id = _figure_out_option(args.result_id, self.input_file) - args.test = _figure_out_option(args.test, self.input_file) - args.version = _figure_out_option(args.version, self.input_file) - - self.logger.info( - f"Checking if result with test={args.test} and result_id={args.result_id} is already there" - ) - json_data = json.dumps( - { - "query": { - "bool": { - "filter": [ - {"term": {"test.keyword": args.test}}, - {"term": {"result_id.keyword": args.result_id}}, - ] - } - } - } - ) - headers = {"Content-Type": "application/json"} - current_doc_in_es = requests.get( - f"{args.base_url}/{args.index}/_search", - headers=headers, - data=json_data, - ) - _check_response(self.logger, current_doc_in_es) - current_doc_in_es = current_doc_in_es.json() - - if current_doc_in_es["hits"]["total"]["value"] > 0: - print( - f"Result test={args.test} and result_id={args.result_id} already there, skipping upload" - ) - return - - logging.info("Uploading result to Results Dashboard") - - upload_data = { - "date": args.date.isoformat(), - "group": args.group, - "link": args.link, - "product": args.product, - "release": args.release, - "result": args.result, - "result_id": args.result_id, - "test": args.test, - "version": args.version, - } - response = requests.post( - f"{args.base_url}/{args.index}/_doc", - headers=headers, - json=upload_data, - ) - _check_response(self.logger, response) - - print(f"Uploaded: {response.content}") - - def set_args(self, parser, subparsers): - parser.add_argument( - "--base-url", - default="http://elasticsearch.intlab.perf-infra.lab.eng.rdu2.redhat.com", - help="Results Dashboard backend url", - ) - parser.add_argument( - "--index", - default="results-dashboard-data", - help="Results Dashboard index where the results are stored", - ) - - # Options for uploading document to Results Dashboard - parser_upload = subparsers.add_parser( - "upload", - help="Upload result to Results Dashboard if it is not there already", - ) - parser_upload.set_defaults(func=self.upload) - parser_upload.add_argument( - "--test", - required=True, - help="Name of the CPT test, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--result-id", - required=True, - help="Test run identifier, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--result", - required=True, - help="Result of the test (choose from: TODO), if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--date", - required=True, - help="When the test ran (ISO 8601 format), if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--link", - required=True, - help="Link to more details, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--group", - required=True, - help="Name of the group where the product belongs, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--product", - required=True, - help="Name of the product, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--release", - default="latest", - help="Release or version stream this result belongst to (this is a way how to group results for multiple versions), if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--version", - default="1", - help="Version of the product on which the test ran, if prefixed with '@' sign, it is a field name from input file where to load this", - ) - parser_upload.add_argument( - "--input-file", - help="If you want to load some values from file, this is the file to specify", - ) - - -class pluginHtml(pluginBase): - def links(self, args): - self.logger.info("Downloading {args.url}") - doc = requests.get(args.url) - - # Regular expression to find all href attributes within tags - # Cheatsheet: - # (?: ... ): Non-capturing group. It groups the pattern inside, but - # doesn't store the matched text as a separate capture group. - # [^>]*?: Matches anything except '>' zero or more times. The '?' - # makes it a non-greedy match so we do not consume href as well. - href_regex = r']*?\s+)?href=["\']([^"\']*)["\']' - matched_hrefs = re.findall(href_regex, doc.text, re.IGNORECASE) - self.logger.info(f"Found {len(matched_hrefs)} links") - - # Filter links down as per user provided expression - compiled_regex = re.compile(args.regexp) - filtered_hrefs = [href for href in matched_hrefs if compiled_regex.match(href)] - self.logger.info(f"Filtered links down to {len(filtered_hrefs)} links") - - # Make all the links absolute to initial user provided URL. - # If there is already absolute link like 'http://www.example.com', - # it will not be affected - absolute_hrefs = [urllib.parse.urljoin(args.url, href) for href in filtered_hrefs] - - # Print what we found - for href in absolute_hrefs: - print(href) - - return absolute_hrefs - - def set_args(self, parser, subparsers): - # Options for listing links from the document - parser_links = subparsers.add_parser( - "links", - help="List links from given HTML document (e.g. httpd directory listing)", - ) - parser_links.set_defaults(func=self.links) - parser_links.add_argument( - "--url", - required=True, - help="HTML document to parse links from", - ) - parser_links.add_argument( - "--regexp", - default=r".*", - help="Only return links matching this regexp (defaults to '.*')", - ) - - -PLUGINS = { - "prow": pluginProw(), - "opensearch": pluginOpenSearch(), - "horreum": pluginHorreum(), - "resultsdashboard": pluginResultsDashboard(), - "html": pluginHtml(), -} - - -def main(): - parser = argparse.ArgumentParser( - description="Shovel data from A to B", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - subparsers = parser.add_subparsers( - dest="plugin_name", - help="sub-command help", - required=True, - ) - for name, plugin in PLUGINS.items(): - parser_plugin = subparsers.add_parser( - name, - help=f"Work with {name}", - ) - subparsers_plugin = parser_plugin.add_subparsers( - dest="plugin_command", - help="sub-command help", - required=True, - ) - plugin.set_args(parser_plugin, subparsers_plugin) - - with skelet.test_setup(parser) as (args, status_data): - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/opl/skelet.py b/opl/skelet.py deleted file mode 100644 index f83700c7..00000000 --- a/opl/skelet.py +++ /dev/null @@ -1,87 +0,0 @@ -import logging -import logging.handlers -import os -import time -from contextlib import contextmanager - -from . import status_data - - -def setup_logger(app_name, stderr_log_lvl): - """ - Create logger that logs to both stderr and log file but with different log levels - """ - # Remove all handlers from root logger if any - logging.basicConfig( - level=logging.NOTSET, handlers=[] - ) # `force=True` was added in Python 3.8 :-( - # Change root logger level from WARNING (default) to NOTSET in order for all messages to be delegated - logging.getLogger().setLevel(logging.NOTSET) - - # Log message format - formatter = logging.Formatter( - "%(asctime)s %(name)s %(threadName)s %(levelname)s %(message)s" - ) - formatter.converter = time.gmtime - - # Silence loggers of some chatty libraries we use - urllib_logger = logging.getLogger("urllib3.connectionpool") - urllib_logger.setLevel(logging.WARNING) - selenium_logger = logging.getLogger("selenium.webdriver.remote.remote_connection") - selenium_logger.setLevel(logging.WARNING) - kafka_logger = logging.getLogger("kafka") - kafka_logger.setLevel(logging.WARNING) - - # Add stderr handler, with provided level - console_handler = logging.StreamHandler() - console_handler.setFormatter(formatter) - console_handler.setLevel(stderr_log_lvl) - logging.getLogger().addHandler(console_handler) - - # Add file rotating handler, with level DEBUG - rotating_handler = logging.handlers.RotatingFileHandler( - filename=f"/tmp/{app_name}.log", maxBytes=100 * 1000, backupCount=2 - ) - rotating_handler.setFormatter(formatter) - rotating_handler.setLevel(logging.DEBUG) - logging.getLogger().addHandler(rotating_handler) - - return logging.getLogger(app_name) - - -@contextmanager -def test_setup(parser, logger_name="root"): - parser.add_argument( - "--status-data-file", - default=os.getenv("STATUS_DATA_FILE", "/tmp/status-data.json"), - help='File where we maintain metadata, results, parameters and measurements for this test run (also use env variable STATUS_DATA_FILE, default to "/tmp/status-data.json")', - ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Show verbose output", - ) - parser.add_argument( - "-d", - "--debug", - action="store_true", - help="Show debug output", - ) - args = parser.parse_args() - - if args.debug: - logger = setup_logger(logger_name, logging.DEBUG) - elif args.verbose: - logger = setup_logger(logger_name, logging.INFO) - else: - logger = setup_logger(logger_name, logging.WARNING) - - logger.debug(f"Args: {args}") - - sdata = status_data.StatusData(args.status_data_file) - - try: - yield (args, sdata) - finally: - sdata.save() diff --git a/opl/status_data.py b/opl/status_data.py deleted file mode 100755 index 102450df..00000000 --- a/opl/status_data.py +++ /dev/null @@ -1,576 +0,0 @@ -import argparse -import copy -import datetime -import json -import logging -import os -import os.path -import pprint -import tempfile - -import deepdiff - -import jinja2 - -import requests - -import tabulate - -import yaml - -from . import cluster_read -from . import date -from . import skelet - - -class StatusData: - def __init__(self, filename, data=None): - self.filename = filename - if filename.startswith("http://") or filename.startswith("https://"): - tmp = tempfile.mktemp() - logging.info( - f"Downloading {filename} to {tmp} and will work with that file from now on" - ) - r = requests.get(filename, verify=False) - with open(tmp, "wb") as fp: - fp.write(r.content) - filename = tmp - - self._filename = filename - self._filename_mtime = None - if data is None: - self.load() - else: - self._data = data - assert "name" in data - assert "started" in data - assert "ended" in data - assert "result" in data - - def load(self): - try: - self._filename_mtime = os.path.getmtime(self._filename) - with open(self._filename, "r") as fp: - self._data = json.load(fp) - logging.debug(f"Loaded status data from {self._filename}") - except FileNotFoundError: - self.clear() - logging.info(f"Opening empty status data file {self._filename}") - - def __getitem__(self, key): - logging.debug(f"Getting item {key} from {self._filename}") - return self._data.get(key, None) - - def __setitem__(self, key, value): - logging.debug(f"Setting item {key} from {self._filename}") - self._data[key] = value - - def __repr__(self): - return f"" - - def __eq__(self, other): - return self._data == other._data - - def __gt__(self, other): - logging.info(f"Comparing {self} to {other}") - return self.get_date("started") > other.get_date("started") - - def _split_mutlikey(self, multikey): - """ - Dots delimits path in the nested dict. - """ - if multikey == "": - return [] - elif multikey.startswith("."): - return multikey[1:].split(".") - else: - return multikey.split(".") - - def _get(self, data, split_key): - if split_key == []: - return data - - if not isinstance(data, dict): - logging.warning( - "Attempted to dive into non-dict. Falling back to return None" - ) - return None - - try: - new_data = data[split_key[0]] - except KeyError: - return None - - if len(split_key) == 1: - return new_data - else: - return self._get(new_data, split_key[1:]) - - def get(self, multikey): - """ - Recursively go through status_data data structure according to - multikey and return its value, or None. For example: - - For example: - - get(('a', 'b', 'c')) - - returns: - - self._data['a']['b']['c'] - - and if say `data['a']['b']` does not exist (or any other key along - the way), return None. - """ - split_key = self._split_mutlikey(multikey) - logging.debug(f"Getting {split_key} from {self._filename}") - return self._get(self._data, split_key) - - def get_date(self, multikey): - i = self.get(multikey) - if i is None: - logging.warning(f"Field {multikey} is None, so can not convert to datetime") - return None - return date.my_fromisoformat(i) - - def _set(self, data, split_key, value): - current_key = split_key[0] - last_key = len(split_key) == 1 - array_key = current_key.endswith("[]") - if array_key: - current_key = current_key[:-2] - missing_key = current_key not in data - - # Check that array key is only used if this is last sub-key - if array_key: - assert last_key, "Arrays can only be last in the multi keys (i.e. 'aaa.bbb[]', but not 'aaa[]'.bbb)" - - # Check that we are not attempting to change type of already existing key - if array_key and not missing_key: - assert type(data[current_key]) == list, "You are trying to change type (e.g. 'aaa' was string and now you are trying to add to 'aaa[]')" - - if missing_key: - if last_key: - if array_key: - data[current_key] = [value] - else: - data[current_key] = value - return # This was last key, we are done - else: - data[current_key] = {} # This is not last key, so it can not be array - return self._set(data[current_key], split_key[1:], value) - else: - if last_key: - if array_key: - data[current_key].append(value) - else: - data[current_key] = value - return # This was last key, we are done - else: - return self._set(data[current_key], split_key[1:], value) # This is not last key, so no need to check for array - - def set(self, multikey, value): - """ - Recursively go through status_data data structure and set value for - multikey. For example: - - set('a.b.c', 123) - - set: - - self._data['a']['b']['c'] = 123 - - even if `self._data['a']['b']` do not exists - then it is created as - empty dict. - - It also supports adding data to lists: - - set('a.b[]', 1) - set('a.b[]', 2) - - results in: - - self._data['a']['b'] = [1, 2] - """ - split_key = self._split_mutlikey(multikey) - logging.debug(f"Setting {'.'.join(split_key)} in {self._filename} to {value}") - if isinstance(value, datetime.datetime): - value = value.isoformat() # make it a string with propper format - self._set(self._data, split_key, copy.deepcopy(value)) - - def set_now(self, multikey): - """ - Set given multikey to current datetime - """ - now = date.get_now() - return self.set(multikey, now.isoformat()) - - def set_subtree_json(self, multikey, file_path): - """ - Set given multikey to contents of JSON formated file provided by its path - """ - with open(file_path, "r") as fp: - if file_path.endswith(".json"): - data = json.load(fp) - elif file_path.endswith(".yaml"): - data = yaml.load(fp, Loader=yaml.SafeLoader) - else: - raise Exception( - f"Unrecognized extension of file to import: {file_path}" - ) - return self.set(multikey, data) - - def _remove(self, data, split_key): - try: - new_data = data[split_key[0]] - except KeyError: - return - - if len(split_key) == 1: - del data[split_key[0]] - return - else: - return self._remove(new_data, split_key[1:]) - - def remove(self, multikey): - """ - Remove given multikey (and it's content) from status data file - """ - split_key = self._split_mutlikey(multikey) - logging.debug(f"Removing {split_key} from {self._filename}") - self._remove(self._data, split_key) - - def list(self, multikey): - """ - For given path, return list of all existing paths below this one - """ - out = [] - split_key = self._split_mutlikey(multikey) - logging.debug(f"Listing {split_key}") - for k, v in self._get(self._data, split_key).items(): - key = ".".join(list(split_key) + [k]) - if isinstance(v, dict): - out += self.list(key) - else: - out.append(key) - return out - - def clear(self): - """ - Default structure - """ - self._data = { - "name": None, - "started": date.get_now_str(), - "ended": None, - "owner": None, - "result": None, - "results": {}, - "parameters": {}, - "measurements": {}, - } - - def info(self): - out = "" - out += f"Filename: {self._filename}\n" - for k, v in self._data.items(): - if not isinstance(v, dict): - out += f"{k}: {v}\n" - return out - - def dump(self): - return self._data - - def save(self, filename=None): - """Save this status data document. - - It makes sure that on disk file was not modified since we loaded it, - but if you provide a filename, this check is skipped. - """ - if filename is None: - if self._filename_mtime is not None: - current_mtime = os.path.getmtime(self._filename) - if self._filename_mtime != current_mtime: - tmp = tempfile.mktemp() - self._save(tmp) - raise Exception( - f"Status data file {self._filename} was modified since we loaded it so I do not want to overwrite it. Instead, saved to {tmp}" - ) - else: - self._filename = filename - - self._save(self._filename) - - def _save(self, filename): - """Just save status data document to JSON file on disk""" - with open(filename, "w+") as fp: - json.dump(self.dump(), fp, sort_keys=True, indent=4) - if filename == self._filename: - self._filename_mtime = os.path.getmtime(filename) - logging.debug(f"Saved status data to {filename}") - - -def doit_set(status_data, set_this): - for item in set_this: - if item == "": - logging.warning("Got empty key=value pair to set - ignoring it") - continue - - key, value = item.split("=") - - if len(value) >= 2 and value[0] == '"' and value[-1] == '"': - value = value[1:-1] - - if value == "%NOW%": - value = date.get_now_str() - else: - try: - value = int(value) - except ValueError: - try: - value = float(value) - except ValueError: - pass - - logging.debug(f"Setting {key} = {value} ({type(value)})") - status_data.set(key, value) - - -def doit_remove(status_data, remove_this): - for item in remove_this: - status_data.remove(item) - - -def doit_set_subtree_json(status_data, set_this): - for item in set_this: - if item == "": - logging.warning("Got empty key=value pair to set - ignoring it") - continue - - key, value = item.split("=") - - logging.debug(f"Setting {key} = {value} (JSON file)") - status_data.set_subtree_json(key, value) - - -def doit_print_oneline(status_data, get_this, get_rounding, get_delimiter): - if not get_rounding: - print(get_delimiter.join([str(status_data.get(i)) for i in get_this])) - else: - for i in get_this: - if isinstance(status_data.get(i), float): - print("{:.2f}".format(status_data.get(i)), end=get_delimiter) - else: - print("{}".format(status_data.get(i)), end=get_delimiter) - print() - - -def doit_additional(status_data, additional, monitoring_start, monitoring_end, args): - requested_info = cluster_read.RequestedInfo( - additional, - start=monitoring_start, - end=monitoring_end, - args=args, - sd=status_data, - ) - - counter_ok = 0 - counter_bad = 0 - for k, v in requested_info: - if k is None: - counter_bad += 1 - else: - status_data.set(k, v) - counter_ok += 1 - - print( - f"Gathered {counter_ok} `ok` data points. Not gathered {counter_bad} `bad` data points" - ) - - -def doit_info(status_data): - print(status_data.info()) - - -def main(): - parser = argparse.ArgumentParser( - description="Work with status data file", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--set", - nargs="*", - default=[], - help='Set key=value data. If value is "%%NOW%%", current date&time is added', - ) - parser.add_argument( - "--set-now", nargs="*", default=[], help="Set key to current date&time" - ) - parser.add_argument( - "--set-subtree-json", - nargs="*", - default=[], - help="Set key to structure from json or yaml formated file (detected by *.json or *.yaml file extension)", - ) - parser.add_argument( - "--get", nargs="*", default=[], help="Print value for given key(s)" - ) - parser.add_argument("--remove", nargs="*", default=[], help="Remove given key(s)") - parser.add_argument( - "--additional", - type=argparse.FileType("r"), - help="Gather more info as specified by the cluster_read.py compatible yaml file", - ) - parser.add_argument( - "--monitoring-start", - type=date.my_fromisoformat, - help="Start of monitoring interval in ISO 8601 format in UTC with seconds precision", - ) - parser.add_argument( - "--monitoring-end", - type=date.my_fromisoformat, - help="End of monitoring interval in ISO 8601 format in UTC with seconds precision", - ) - parser.add_argument( - "--monitoring-raw-data-dir", - type=cluster_read.dir_path, - help="Provide a direcotory if you want raw monitoring data to be dumped in CSV files form", - ) - parser.add_argument( - "--end", - action="store_true", - help='"started" is set when the status data file is created, "ended" is set when this is used', - ) - parser.add_argument( - "--info", action="store_true", help="Show basic info from status data file" - ) - parser.add_argument( - "--decimal-rounding", - action="store_true", - help="Rounding a number to its hundredths, leaving 2 numbers after decimal point", - ) - parser.add_argument( - "--delimiter", - default="\t", - help='When returning more "--get" fields, delimit them with this (default is tab)', - ) - for name, plugin in cluster_read.PLUGINS.items(): - plugin.add_args(parser) - - with skelet.test_setup(parser) as (args, status_data): - if len(args.set) > 0: - doit_set(status_data, args.set) - if len(args.set_now) > 0: - doit_set(status_data, [k + "=%NOW%" for k in args.set_now]) - if len(args.set_subtree_json) > 0: - doit_set_subtree_json(status_data, args.set_subtree_json) - if len(args.get) > 0: - doit_print_oneline( - status_data, args.get, args.decimal_rounding, args.delimiter - ) - if len(args.remove) > 0: - doit_remove(status_data, args.remove) - if args.additional: - doit_additional( - status_data, - args.additional, - args.monitoring_start, - args.monitoring_end, - args, - ) - if args.end: - doit_set(status_data, ["ended=%NOW%"]) - if args.info: - doit_info(status_data) - - -def main_diff(): - parser = argparse.ArgumentParser( - description="Compare two status data files", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("first", nargs=1, help="First file to compare") - parser.add_argument("second", nargs=1, help="Second file to compare") - parser.add_argument("--report", action="store_true", help="Show formated report") - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - logging.debug(f"Args: {args}") - - first = StatusData(args.first[0]) - second = StatusData(args.second[0]) - - diff = deepdiff.DeepDiff(first._data, second._data, view="tree") - if args.report: - print(f"Keys: {', '.join(diff.keys())}") - if "dictionary_item_added" in diff: - print("\nDictionary items added:\n") - table = [] - for i in diff["dictionary_item_added"]: - table.append([i.path(), i.t2]) - print(tabulate.tabulate(table, headers=["path", "added value"])) - if "dictionary_item_removed" in diff: - print("\nDictionary items removed:\n") - table = [] - for i in diff["dictionary_item_removed"]: - table.append([i.path(), i.t1]) - print(tabulate.tabulate(table, headers=["path", "removed value"])) - if "values_changed" in diff: - print("\nValues changed:\n") - table = [] - for i in diff["values_changed"]: - d = None - try: - first = float(i.t1) - second = float(i.t2) - d_raw = (second - first) / first * 100 - if abs(d_raw) < 1: - d = f"{d_raw:.3f}" - else: - d = f"{d_raw:.0f}" - except (ValueError, ZeroDivisionError): - pass - table.append([i.path(), i.t1, i.t2, d]) - print( - tabulate.tabulate( - table, headers=["path", "first", "second", "change [%]"] - ) - ) - if "type_changes" in diff: - print("\nTypes changed:\n") - table = [] - for i in diff["type_changes"]: - table.append([i.path(), type(i.t1), type(i.t2)]) - print(tabulate.tabulate(table, headers=["path", "first", "second"])) - else: - pprint.pprint(diff) - - -def main_report(): - parser = argparse.ArgumentParser( - description="Create a report using provided template from status" " data file", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("template", help="Report template file to use") - parser.add_argument("status_data", help="Status data file to format") - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - logging.debug(f"Args: {args}") - - # Load Jinja2 template - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.dirname(args.template)) - ) - template = env.get_template(os.path.basename(args.template)) - - # Load status data document - data = StatusData(args.status_data) - - print(template.render({"data": data})) diff --git a/opl/status_data_report.txt b/opl/status_data_report.txt deleted file mode 100644 index 3de78e52..00000000 --- a/opl/status_data_report.txt +++ /dev/null @@ -1,3 +0,0 @@ -Test name: {{ data.get('name') }} - start: {{ data.get('started') }} - end: {{ data.get('ended') }} diff --git a/opl/status_data_updater.py b/opl/status_data_updater.py deleted file mode 100755 index 38bcdc13..00000000 --- a/opl/status_data_updater.py +++ /dev/null @@ -1,884 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import datetime -import json -import logging -import os -import random -import tempfile -import time -from collections import OrderedDict - -import opl.status_data - -import requests -import requests.adapters - -import tabulate - -import urllib3 - -import yaml - - -RP_TO_ES_STATE = { - "automation_bug": "FAIL", - "no_defect": "PASS", - "product_bug": "FAIL", - "system_issue": "ERROR", - "to_investigate": "FAIL", -} - -STATE_WEIGHTS = { - "PASS": 0, - "FAIL": 1, - "ERROR": 2, -} - - -def get_session(): - session = requests.Session() - retry_adapter = requests.adapters.HTTPAdapter( - max_retries=urllib3.Retry(total=None, connect=10, backoff_factor=1) - ) - session.mount("https://", retry_adapter) - session.mount("http://", retry_adapter) - return session - - -def _es_get_test(session, args, key, val, size=1, sort_by="started", sort_order="desc"): - url = f"{args.es_server}/{args.es_index}/_search" - headers = { - "Content-Type": "application/json", - } - data = { - "query": { - "bool": { - "filter": [], - }, - }, - "sort": { - sort_by: { - "order": sort_order, - }, - }, - "size": size, - } - - for k, v in zip(key, val): - data["query"]["bool"]["filter"].append( - { - "term": { - k: v, - }, - } - ) - - if session is None: - session = get_session() - - logging.info( - f"Querying ES with url={url}, headers={headers} and json={json.dumps(data)}" - ) - attempt = 0 - attempt_max = 10 - while True: - try: - response = session.get(url, headers=headers, json=data) - except requests.exceptions.ConnectionError: - if attempt >= attempt_max: - raise - attempt += 1 - time.sleep(attempt) - else: - break - response.raise_for_status() - logging.debug( - f"Got back this: {json.dumps(response.json(), sort_keys=True, indent=4)}" - ) - - return response.json() - - -def _add_comment(args, sd, author=None, text=None): - """Add text as a comment to status data document.""" - if sd.get("comments") is None: - sd.set("comments", []) - - if not isinstance(sd.get("comments"), list): - logging.error(f"Field 'comments' is not a list: {sd.get('comments')}") - - if author is None: - author = os.getenv("USER", "unknown") - if text is None: - text = "Setting " + ", ".join(args.change_set) - - sd.get("comments").append( - { - "author": author, - "date": datetime.datetime.utcnow() - .replace(tzinfo=datetime.timezone.utc) - .isoformat(), - "text": text, - } - ) - - -def doit_list(args): - assert args.list_name is not None - - response = _es_get_test( - None, args, ["name.keyword"], [args.list_name], args.list_size - ) - - table_headers = [ - "Run ID", - "Started", - "Owner", - "Golden", - "Result", - ] + args.list_fields - table = [] - - for item in response["hits"]["hits"]: - sd = _create_sd_from_es_response(item) - row = [ - sd.get("id"), - sd.get("started"), - sd.get("owner"), - sd.get("golden"), - sd.get("result"), - ] - row += [sd.get(i) for i in args.list_fields] - table.append(row) - - print(tabulate.tabulate(table, headers=table_headers)) - - -def doit_change(args): - assert args.change_id is not None - - response = _es_get_test(None, args, ["id.keyword"], [args.change_id]) - - source = response["hits"]["hits"][0] - es_type = source["_type"] - es_id = source["_id"] - sd = _create_sd_from_es_response(source) - - for item in args.change_set: - if item == "": - logging.warning("Got empty key=value pair to set - ignoring it") - continue - - key, value = item.split("=") - - try: - value = int(value) - except ValueError: - try: - value = float(value) - except ValueError: - pass - - logging.debug(f"Setting {key} = {value} ({type(value)})") - sd.set(key, value) - - # Add comment to log the change - _add_comment(args, sd, text=args.change_comment_text) - - url = f"{args.es_server}/{args.es_index}/{es_type}/{es_id}" - - logging.info(f"Saving to ES with url={url} and json={json.dumps(sd.dump())}") - - if args.dry_run: - logging.info("Not touching ES as we are running in dry run mode") - else: - response = requests.post(url, json=sd.dump()) - response.raise_for_status() - logging.debug( - f"Got back this: {json.dumps(response.json(), sort_keys=True, indent=4)}" - ) - - print(sd.info()) - - -def _get_rp_launches(session, args, rp_launch=None, rp_launches_count=None): - """Get N newest launches from RP""" - if rp_launch is None: - rp_launch = args.rp_launch - if rp_launches_count is None: - rp_launches_count = args.rp_launches_count - - url = f"https://{args.rp_host}/api/v1/{args.rp_project}/launch" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - data = { - "filter.eq.name": rp_launch, - "page.size": rp_launches_count, - "page.sort": "endTime,desc", - } - logging.debug(f"Going to do GET request to {url} with {data}") - response = session.get( - url, params=data, headers=headers, verify=not args.rp_noverify - ) - if not response.ok: - logging.error(f"Request failed: {response.text}") - response.raise_for_status() - logging.debug(f"Request returned {response.json()}") - return response.json()["content"] - - -def _get_run_id_from_rp_launch(launch): - """Return "run_id" attribute value from RP launch or None if not present""" - run_id = None - for a in launch["attributes"]: - if a["key"] == "run_id": - run_id = a["value"] - break - return run_id - - -def _filter_rp_launches_without_run_id(launches): - """Filter out RP launches that does not have "run_id" attribute""" - launches_filtered = [] - for launch in launches: - run_id = _get_run_id_from_rp_launch(launch) - if run_id is None: - logging.warning( - f"Launch id={launch['id']} do not have run_id attribute, skipping it" - ) - continue - launches_filtered.append(launch) - return launches_filtered - - -def _get_rp_launch_results(session, args, launch): - """Get results for RP launch""" - results = [] - url = f"https://{args.rp_host}/api/v1/{args.rp_project}/item" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - data = { - "filter.eq.launchId": launch["id"], - "filter.eq.type": "TEST", - "filter.ne.status": "PASSED", - "page.size": 100, - "page.page": 0, - "page.sort": "id,asc", - } - while True: - logging.debug(f"Going to do GET request to {url} with {data}") - response = session.get( - url, params=data, headers=headers, verify=not args.rp_noverify - ) - results += response.json()["content"] - if response.json()["page"]["number"] < response.json()["page"]["totalPages"]: - data["page.page"] += 1 - else: - logging.debug( - "No content in the response, considering this last page of data" - ) - break - logging.debug(f"OK, we have {len(results)} results from RP for this launch") - return results - - -def _create_sd_from_es_response(response): - """Convert ElasticSearch response data structure to StatusData object.""" - logging.debug( - f"Loading data from document ID {response['_id']} with field id={response['_source']['id'] if 'id' in response['_source'] else None}" - ) - tmpfile = tempfile.NamedTemporaryFile(prefix=response["_id"], delete=False).name - return opl.status_data.StatusData(tmpfile, data=response["_source"]) - - -def _get_es_result_for_rp_result(session, args, run_id, result): - if args.rp_project == "satcpt": - # OK, I agree we need a better way here. - # In all projects except SatCPT we have 1 run_id for 1 test - # result, but in SatCPT we need to differentiate by name as - # well and that is composed differently in SatCPT and in other - # CPTs :-( - if "itemPaths" not in result["pathNames"]: - raise Exception( - f"This result do not have result -> pathNames -> itemPaths, skipping it: {result}" - ) - else: - sd_name = f"{result['pathNames']['itemPaths'][0]['name']}/{result['name']}" - response = _es_get_test( - session, args, ["id.keyword", "name.keyword"], [run_id, sd_name] - ) - elif args.rp_project == "aapcpt": - response = _es_get_test( - session, args, ["id.keyword", "name.keyword"], [run_id, result["name"]] - ) - else: - response = _es_get_test(session, args, ["id.keyword"], [run_id]) - assert response["hits"]["total"]["value"] == 1 - try: - source = response["hits"]["hits"][0] - except IndexError: - raise Exception(f"Failed to find test result in ES for {run_id}") - es_type = source["_type"] - es_id = source["_id"] - sd = _create_sd_from_es_response(source) - return (sd, es_type, es_id) - - -def _get_es_dashboard_result_for_run_id(session, args, run_id, test=None): - if test is not None: - response = _es_get_test( - session, - args, - ["result_id.keyword", "test.keyword"], - [run_id, test], - sort_by="date", - sort_order="asc", - ) - else: - response = _es_get_test( - session, - args, - ["result_id.keyword"], - [run_id], - sort_by="date", - sort_order="asc", - ) - if response["hits"]["total"]["value"] == 0: - return (None, None, None) - try: - # In combination with _es_get_test()'s sort_order="asc", get the oldest result. - source = response["hits"]["hits"][0] - except IndexError: - logging.debug(f"Failed to find dashboard result in ES for {run_id}") - return (None, None, None) - else: - return (response["hits"]["hits"][0]["_source"], source["_type"], source["_id"]) - - -def _get_rp_result_defect_string(result): - return list(result["statistics"]["defects"].keys())[0] - - -def _get_rp_result_result_string(result): - return RP_TO_ES_STATE[_get_rp_result_defect_string(result)] - - -def doit_rp_to_es(args): - assert args.es_server is not None - assert args.rp_host is not None - - stats = { - "launches": 0, - "cases": 0, - "cases_changed": 0, - } - - if args.rp_noverify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Start a session - session = get_session() - - # Get 10 newest launches - launches = _get_rp_launches(session, args) - - # Filter out RP launches that does not have "run_id" attribute - launches = _filter_rp_launches_without_run_id(launches) - - for launch in launches: - stats["launches"] += 1 - - # Get run ID from launch attributes - run_id = _get_run_id_from_rp_launch(launch) - - # Get test results from launch - results = _get_rp_launch_results(session, args, launch) - print(f"Going to compare {len(results)} results for launch {launch['id']}") - - # Process individual results - for result in results: - logging.debug(f"Processing RP result {result}") - stats["cases"] += 1 - - # Get resuls from launch statistics - result_string = _get_rp_result_result_string(result) - - # Get relevant status data document from ElasticSearch - try: - sd, es_type, es_id = _get_es_result_for_rp_result( - session, args, run_id, result - ) - except Exception as e: - logging.warning( - f"Something went wrong when getting data for {run_id}/{result}: {e}" - ) - continue - - logging.debug( - f"Comparing result from RP {result_string} to result from ES {sd.get('result')}" - ) - if sd.get("result") != result_string: - stats["cases_changed"] += 1 - - # Add comment to log the change - try: - comment = "Comment from RP: " + result["issue"]["issueType"] - except IndexError: - comment = f"Automatic update as per ReportPortal change: {sd.get('result')} -> {result_string}" - _add_comment(args, sd, author="status_data_updater", text=comment) - - logging.info( - f"Results do not match, updating them: {sd.get('result')} != {result_string}" - ) - sd.set("result", result_string) - - # Save the changes to ES - url = f"{args.es_server}/{args.es_index}/{es_type}/{es_id}" - logging.info( - f"Saving to ES with url={url} and json={json.dumps(sd.dump())}" - ) - if args.dry_run: - logging.info("Not touching ES as we are running in dry run mode") - else: - attempt = 0 - attempt_max = 10 - while True: - response = session.post(url, json=sd.dump()) - if ( - response.status_code == 429 - ): # 429 Client Error: Too Many Requests for url: http://...//_doc/... - attempt += 1 - if attempt >= attempt_max: - raise Exception( - f"Failed to update data in ES after {attempt} attempts: {response}" - ) - else: - logging.info( - f"Request failed with '429 Client Error: Too Many Requests'. Will retry in a bit. Attempt {attempt}/{attempt_max}" - ) - time.sleep(random.randint(1, 10)) - else: - break - response.raise_for_status() - logging.debug( - f"Got back this: {json.dumps(response.json(), sort_keys=True, indent=4)}" - ) - - print(tabulate.tabulate(stats.items())) - - -def doit_rp_to_dashboard_new(args): - assert args.es_server is not None - - if args.rp_noverify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Start a session - session = get_session() - - run_id = args.dashboard_run_id - result = args.dashboard_result - - if not args.dashboard_skip_uniqness_check: - # Ensure there are no results for this run_id in ElasticSearch yet - try: - dashboard, es_type, es_id = _get_es_dashboard_result_for_run_id( - session, args, run_id - ) - except requests.exceptions.HTTPError as e: - matching = ( - "No mapping found for [date] in order to sort on" in e.response.text - ) - if e.response.status_code == 400 and matching: - logging.debug( - "Request failed, but I guess it was because index is still empty" - ) - dashboard = None - else: - raise - assert dashboard is None, f"Result {run_id} already exists: {dashboard}" - - # Create new result in the dashboard - url = f"{args.es_server}/{args.es_index}/_doc/" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - data = { - "result_id": run_id, - "group": args.dashboard_group, - "product": args.dashboard_product, - "release": args.dashboard_release, - "version": args.dashboard_version, - "link": args.dashboard_link, - "test": args.dashboard_test, - "result": result, - "date": args.dashboard_date, - } - logging.debug(f"Going to do POST request to {url} with {data}") - response = session.post( - url, json=data, headers=headers, verify=not args.rp_noverify - ) - response.raise_for_status() - logging.debug( - f"Got back this: {json.dumps(response.json(), sort_keys=True, indent=4)}" - ) - print(f"Created result {run_id} in the dashboard with value {result}") - - -def _update_es_dashboard_result(session, args, es_id, result_string): - url = f"{args.es_server}/{args.es_index}/_doc/{es_id}/_update" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.rp_token}", - } - data = { - "doc": { - "result": result_string, - }, - } - logging.debug(f"Going to do POST request to {url} with {data}") - if args.dry_run: - logging.debug("Skipped because of dry-run") - else: - attempt = 0 - attempt_max = 10 - while True: - try: - response = session.post( - url, json=data, headers=headers, verify=not args.rp_noverify - ) - except requests.exceptions.ConnectionError: - if attempt >= attempt_max: - raise - attempt += 1 - time.sleep(attempt) - else: - break - response.raise_for_status() - logging.debug( - f"Got back this: {json.dumps(response.json(), sort_keys=True, indent=4)}" - ) - - -def doit_rp_to_dashboard_update(args): - assert args.es_server is not None - assert args.rp_host is not None - - stats = { - "launches": 0, - "results": 0, - "results_changed": 0, - } - - if args.rp_noverify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Start a session - session = requests.Session() - - # Get 10 newest launches - launches = _get_rp_launches(session, args) - - # Filter out RP launches that does not have "run_id" attribute - launches = _filter_rp_launches_without_run_id(launches) - - for launch in launches: - stats["launches"] += 1 - - # Get run ID from launch attributes - run_id = _get_run_id_from_rp_launch(launch) - - # Get test results from launch - results = _get_rp_launch_results(session, args, launch) - print(f"Going to compare {len(results)} results for launch {launch['id']}") - - # Process individual results to get final result - for result in results: - logging.debug(f"Processing RP result {result}") - - result_string = _get_rp_result_result_string(result) - - # Get relevant dashboard result from ElasticSearch - dashboard, es_type, es_id = _get_es_dashboard_result_for_run_id( - session, - args, - run_id, - result["name"], - ) - if dashboard is None: - logging.warning( - f"Result {run_id} '{result['name']}' does not exist in the dashboard, skipping updating it" - ) - continue - - # Update the result in dashboard if needed - stats["results"] += 1 - if dashboard["result"] == result_string: - pass # data in the dashboard are correct, no action needed - else: - _update_es_dashboard_result( - session, - args, - es_id, - result_string, - ) - stats["results_changed"] += 1 - - print(tabulate.tabulate(stats.items())) - - -def doit_rp_backlog(args): - assert args.rp_host is not None - assert args.jobs_ownership_config is not None - - if args.rp_noverify: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - # Start a session - session = requests.Session() - - with open(args.jobs_ownership_config, "r") as fp: - launches_to_check = yaml.load(fp, Loader=yaml.Loader) - - data_per_owner = OrderedDict() - data_per_job = OrderedDict() - - for launch_to_check in launches_to_check: - rp_launch = launch_to_check["name"] - rp_launches_count = launch_to_check["history"] - rp_launch_owner = launch_to_check["owner"] - if rp_launch_owner not in data_per_owner: - data_per_owner[rp_launch_owner] = OrderedDict( - [ - ("automation_bug", 0), - ("no_defect", 0), - ("product_bug", 0), - ("system_issue", 0), - ("to_investigate", 0), - ] - ) - if rp_launch not in data_per_job: - data_per_job[rp_launch] = OrderedDict( - [ - ("automation_bug", 0), - ("no_defect", 0), - ("product_bug", 0), - ("system_issue", 0), - ("to_investigate", 0), - ] - ) - - # Get N newest launches - launches = _get_rp_launches( - session, args, rp_launch=rp_launch, rp_launches_count=rp_launches_count - ) - - # Filter out RP launches that does not have "run_id" attribute - launches = _filter_rp_launches_without_run_id(launches) - - for launch in launches: - # Get run ID for a launch - run_id = _get_run_id_from_rp_launch(launch) - - # Get test results from launch - results = _get_rp_launch_results(session, args, launch) - logging.debug( - f"Going to compare {len(results)} results for launch {launch['id']}" - ) - - # Process individual results - for result in results: - logging.debug(f"Processing RP result {result}") - - # Get resuls from launch statistics - assert ( - result["statistics"]["executions"]["total"] == 1 - ), "We only know how to work with results with one executions" - result_string = result["status"] - defect_string = _get_rp_result_defect_string(result) - override_string = _get_rp_result_result_string(result) - logging.debug( - f"Counted result {run_id}: {result_string}, {defect_string}, {override_string}" - ) - data_per_owner[rp_launch_owner][defect_string] += 1 - data_per_job[rp_launch][defect_string] += 1 - - headers = sorted(list(data_per_owner[next(iter(data_per_owner.keys()))].keys())) - - rows = [] - for owner in data_per_owner: - row = [owner] - for defect in headers: - row.append(data_per_owner[owner].get(defect, None)) - rows.append(row) - print(tabulate.tabulate(rows, headers=headers)) - - rows = [] - for job in data_per_job: - row = [job] - for defect in headers: - row.append(data_per_job[job].get(defect, None)) - rows.append(row) - print(tabulate.tabulate(rows, headers=headers)) - - -def main(): - parser = argparse.ArgumentParser( - description="Investigate and modify status data documents in ElasticSearch", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--action", - required=True, - choices=[ - "list", - "change", - "rp-to-es", - "rp-to-dashboard-new", - "rp-to-dashboard-update", - "rp-backlog", - ], - help="What action to do", - ) - - parser.add_argument( - "--es-server", - default="http://elasticsearch.example.com:9286", - help="ElasticSearch server for the results data", - ) - parser.add_argument( - "--es-index", - default="my-index", - help="ElasticSearch index for the results data", - ) - - parser.add_argument( - "--list-name", help="Name of the test to query for when listing" - ) - parser.add_argument( - "--list-size", - type=int, - default=50, - help="Number of documents to show when listing", - ) - parser.add_argument( - "--list-fields", - nargs="+", - default=[], - help="Additional fields to add to the table", - ) - - parser.add_argument("--change-id", help="ID of a test run when changing") - parser.add_argument( - "--change-set", nargs="*", default=[], help="Set key=value data" - ) - parser.add_argument( - "--change-comment-text", help="Comment to be added as part of change" - ) - - parser.add_argument("--rp-host", help="ReportPortal host") - parser.add_argument( - "--rp-noverify", - action="store_true", - help="When talking to ReportPortal ignore certificate verification failures", - ) - parser.add_argument("--rp-project", help="ReportPortal project") - parser.add_argument("--rp-token", help="ReportPortal token") - parser.add_argument("--rp-launch", help="ReportPortal launch name") - parser.add_argument( - "--rp-launches-count", - default=10, - type=int, - help="Number of ReportPortal launches to load", - ) - - parser.add_argument( - "--jobs-ownership-config", - help="YAML config with owners and history size for ReportPortal 'to_investigate' rp-backlog feature", - ) - - parser.add_argument( - "--dashboard-run-id", - default="Unknown run_id", - help="When pushing new result to dashboard, this is the run_id", - ) - parser.add_argument( - "--dashboard-result", - default="ERROR", - help="When pushing new result to dashboard, this is the result", - ) - parser.add_argument( - "--dashboard-test", - default="N/A", - help="When pushing new result to dashboard, this is the test name", - ) - parser.add_argument( - "--dashboard-group", - default="Unknown group", - help="Product group for result dashboard", - ) - parser.add_argument( - "--dashboard-product", - default="Unknown product", - help="Product for result dashboard", - ) - parser.add_argument( - "--dashboard-release", - default="Unknown release", - help="Product release stream for result dashboard", - ) - parser.add_argument( - "--dashboard-version", - default="Unknown version", - help="Application version during the test for result dashboard", - ) - parser.add_argument( - "--dashboard-link", - default="Unknown link", - help="Link with test details for result dashboard", - ) - parser.add_argument( - "--dashboard-date", - default=datetime.datetime.utcnow() - .replace(tzinfo=datetime.timezone.utc) - .isoformat(), - help="When the test was executed for result dashboard", - ) - parser.add_argument( - "--dashboard-skip-uniqness-check", - action="store_true", - help="Do not check that result with this run ID exists in result dashboard", - ) - - parser.add_argument( - "--dry-run", - action="store_true", - help="Do not actually change data, meant for debugging", - ) - parser.add_argument("-d", "--debug", action="store_true", help="Show debug output") - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.DEBUG) - - logging.debug(f"Args: {args}") - - if args.action == "list": - return doit_list(args) - if args.action == "change": - return doit_change(args) - if args.action == "rp-to-es": - return doit_rp_to_es(args) - if args.action == "rp-to-dashboard-new": - return doit_rp_to_dashboard_new(args) - if args.action == "rp-to-dashboard-update": - return doit_rp_to_dashboard_update(args) - if args.action == "rp-backlog": - return doit_rp_backlog(args) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..48299426 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=core/opl --cov=extras/opl --cov-report=term-missing --cov-report=html diff --git a/setup.py b/setup.py index d66f6dcb..b9b08546 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,12 @@ -#!/usr/bin/env python3 - +import os import setuptools with open("README.md", "r") as fh: long_description = fh.read() +core_path = os.path.abspath("core") +extras_path = os.path.abspath("extras") + setuptools.setup( name="opl-rhcloud-perf-team", version="0.0.1", @@ -14,7 +16,7 @@ long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/redhat-performance/opl", - packages=setuptools.find_packages(), + packages=[], # Explicitly empty classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License (GPL)", @@ -24,69 +26,17 @@ ], python_requires=">=3.6", install_requires=[ - "Jinja2>=3.0", - "boto3", - "junitparser", - "kafka-python", - "locust", - "psycopg2-binary", - "PyYAML", - "requests", - "tabulate", - "deepdiff", - "numpy", - # Other package dependencies + f"opl-rhcloud-perf-team-core @ file://{core_path}", + f"opl-rhcloud-perf-team-extras @ file://{extras_path}", ], extras_require={ "dev": [ "pytest", + "pytest-cov", "black", "flake8", "pyfakefs", - # Other development dependencies + "responses", ] }, - package_data={ - "opl": [ - "status_data_report.txt", - "cluster_read_example.yaml", - "cluster_read_sat.yaml", - ], - "opl.generators": [ - "inventory_egress_data.json", - "inventory_egress_template.json.j2", - "inventory_ingress_RHSM_template.json.j2", - "inventory_ingress_puptoo_template.json.j2", - "inventory_ingress_yupana_template.json.j2", - "inventory_ingress_InvGitUtilsPayload_template.json.j2", - "packages_data.json", - "enabled_services.txt", - "installed_services.txt", - "running_processes.txt", - "yum_repos.txt", - "chrome_notifications_template.json.j2", - ], - "opl.investigator": [ - "sample_config.yaml", - ], - }, - entry_points={ - "console_scripts": [ - "cluster_read.py = opl.cluster_read:main", - "pass_or_fail.py = opl.pass_or_fail:main", - "junit_cli.py = opl.junit_cli:main", - "rp_updater.py = opl.rp_updater:main", - "status_data_diff.py = opl.status_data:main_diff", - "status_data.py = opl.status_data:main", - "status_data_report.py = opl.status_data:main_report", - "status_data_updater.py = opl.status_data_updater:main", - "script-skip-to-end.py = opl.skip_to_end:main", - "script-manage-db.py = opl.manage_db:main", - "script-hbi-populate.py = opl.hbi_utils:populate_main", - "script-hbi-cleanup.py = opl.hbi_utils:cleanup_main", - "shovel.py = opl.shovel:main", - "create_packages_template_from_dnf_repoquery.py = opl.create_packages_template_from_dnf_repoquery:parse_repoquery_output_from_stdin", - "horreum_api.py = opl.horreum_api:main", - ], - }, ) diff --git a/tests/test_generators.py b/tests/test_generators.py index 276db223..9420f026 100644 --- a/tests/test_generators.py +++ b/tests/test_generators.py @@ -12,29 +12,35 @@ class TestGenericGenerator(unittest.TestCase): def test_count(self): _, template = tempfile.mkstemp(dir=os.path.dirname(opl.generators.generic.__file__), text=True) - with open(template, "w") as fd: - fd.write('{"foo": "bar"}') + try: + with open(template, "w") as fd: + fd.write('{"foo": "bar"}') - gg = opl.generators.generic.GenericGenerator(3, os.path.basename(template)) + gg = opl.generators.generic.GenericGenerator(3, os.path.basename(template)) - counter = 0 - for message in gg: - self.assertIsInstance(message, tuple) - counter += 1 + counter = 0 + for message in gg: + self.assertIsInstance(message, tuple) + counter += 1 - self.assertEqual(counter, 3) + self.assertEqual(counter, 3) + finally: + os.remove(template) def test_uuids(self): _, template = tempfile.mkstemp(dir=os.path.dirname(opl.generators.generic.__file__), text=True) - with open(template, "w") as fd: - fd.write('{"x": "{{ opl_gen.gen_uuid() }}"}') + try: + with open(template, "w") as fd: + fd.write('{"x": "{{ opl_gen.gen_uuid() }}"}') - gg = opl.generators.generic.GenericGenerator(3, os.path.basename(template)) + gg = opl.generators.generic.GenericGenerator(3, os.path.basename(template)) - uuids = [] - for message in gg: - self.assertEqual(len(message[1]["x"]), 36) - uuids.append(message[1]["x"]) + uuids = [] + for message in gg: + self.assertEqual(len(message[1]["x"]), 36) + uuids.append(message[1]["x"]) - self.assertEqual(len(uuids), 3) + self.assertEqual(len(uuids), 3) + finally: + os.remove(template) diff --git a/tests/test_skelet.py b/tests/test_skelet.py index 1e4cb10e..5ca99263 100644 --- a/tests/test_skelet.py +++ b/tests/test_skelet.py @@ -12,5 +12,5 @@ def test_test_setup(self): parser.add_argument( "--fake_arg", help="Just to catch test file name when calling tests" ) - with opl.skelet.test_setup(parser) as (args, status_data): + with opl.skelet.test_setup(parser, args=[]) as (args, status_data): self.assertIn("debug", args)