classDiagram class MapReduceJobResource { <> POST /jobs : JobResponse GET /jobs/:jobId : JobResponse GET /jobs : JobListResponse PUT /jobs/:jobId : JobResponse DELETE /jobs/:jobId : void } class ShufflePhaseResource { <> GET /jobs/:jobId/shuffle : ShuffleMetricsResponse GET /jobs/:jobId/shuffle/partitions : PartitionListResponse GET /jobs/:jobId/shuffle/partitions/:partitionId : PartitionDetailsResponse } class SkewDetectionResource { <> GET /jobs/:jobId/skew : SkewAnalysisResponse GET /jobs/:jobId/skew/partitions/:partitionId : PartitionSkewResponse POST /jobs/:jobId/skew/mitigate : MitigationResponse } class TaskResource { <> GET /jobs/:jobId/tasks : TaskListResponse GET /jobs/:jobId/tasks/:taskId : TaskResponse GET /jobs/:jobId/tasks/:taskId/metrics : TaskMetricsResponse } class ReducerGroupResource { <> GET /jobs/:jobId/reducers : ReducerGroupListResponse GET /jobs/:jobId/reducers/:reducerId : ReducerGroupResponse POST /jobs/:jobId/reducers/:reducerId/rebalance : RebalanceResponse } class JobRequest { <> string jobName string jobType integer mapperCount integer reducerCount string inputPath string outputPath map configuration } class JobResponse { <> string jobId string jobName string status long startTime long endTime integer mapperCount integer reducerCount map metrics } class JobListResponse { <> list~JobResponse~ jobs integer totalCount integer pageNumber } class ShuffleMetricsResponse { <> string jobId long totalShuffleBytes long shuffleDuration float averageShuffle integer partitionCount map partitionMetrics } class PartitionListResponse { <> list~PartitionSummary~ partitions integer totalPartitions } class PartitionSummary { <> integer partitionId long recordCount long dataSize float skewFactor } class PartitionDetailsResponse { <> integer partitionId long recordCount long dataSize list~string~ assignedReducers map keyDistribution long processingTime } class SkewAnalysisResponse { <> string jobId float overallSkewRatio list~SkewedPartition~ skewedPartitions string recommendation long analysisTime } class SkewedPartition { <> integer partitionId float skewFactor long expectedSize long actualSize list~string~ hotKeys } class PartitionSkewResponse { <> integer partitionId float skewFactor map keyFrequency list~string~ topKeys string mitigationStrategy } class MitigationResponse { <> string jobId string mitigationStrategy boolean success string description map newPartitionMap float estimatedImprovement } class TaskListResponse { <> list~TaskSummary~ tasks integer totalTasks string jobId } class TaskSummary { <> string taskId string taskType string status long duration long inputRecords long outputRecords } class TaskResponse { <> string taskId string jobId string taskType string status long startTime long endTime string assignedNode map taskMetadata } class TaskMetricsResponse { <> string taskId long cpuTimeMs long memoryUsageMB long ioReadBytes long ioWriteBytes long shuffleInputBytes long shuffleOutputBytes } class ReducerGroupListResponse { <> list~ReducerGroupSummary~ reducerGroups integer totalReducers } class ReducerGroupSummary { <> string reducerId integer assignedPartitions long totalInputBytes long totalOutputBytes } class ReducerGroupResponse { <> string reducerId string status list~integer~ assignedPartitionIds long inputBytes long outputBytes float processingSpeed string hostNode } class RebalanceResponse { <> string jobId string reducerId boolean success list~integer~ newPartitionAssignment float expectedLoadReduction string rebalanceStrategy } MapReduceJobResource --> JobRequest : submits MapReduceJobResource --> JobResponse : returns MapReduceJobResource --> JobListResponse : returns ShufflePhaseResource --> ShuffleMetricsResponse : returns ShufflePhaseResource --> PartitionListResponse : returns ShufflePhaseResource --> PartitionDetailsResponse : returns PartitionListResponse --> PartitionSummary : contains PartitionDetailsResponse --> JobResponse : references SkewDetectionResource --> SkewAnalysisResponse : returns SkewDetectionResource --> PartitionSkewResponse : returns SkewDetectionResource --> MitigationResponse : returns SkewAnalysisResponse --> SkewedPartition : contains TaskResource --> TaskListResponse : returns TaskResource --> TaskResponse : returns TaskResource --> TaskMetricsResponse : returns TaskListResponse --> TaskSummary : contains ReducerGroupResource --> ReducerGroupListResponse : returns ReducerGroupResource --> ReducerGroupResponse : returns ReducerGroupResource --> RebalanceResponse : returns ReducerGroupListResponse --> ReducerGroupSummary : contains

System design library

Processing & Compute: The MapReduce paradigm (dealing with shuffle-heavy jobs and skew)