Unverified Commit b34b8ff6 authored by Claude's avatar Claude
Browse files

Fix accuracy metrics and remove overall score 

- Fixed accuracy calculation to show % of predictions within 20% of actual (proper metric)
- Renamed overall_error_pct to mape for clarity
- Removed overall_score calculation (was combining accuracy, profit, and service level)
- Updated results template to show MAPE and prediction accuracy side-by-side
- Added explanatory text for both metrics (lower/higher is better)
parent 32842673

forecast_app/app.py

+18 −21
Original line number Diff line number Diff line
@@ -181,33 +181,30 @@ def calculate_results(predictions, test_data): 


    # Calculate overall prediction accuracy

    all_errors = []

    accurate_predictions = 0  # Count predictions within 20% of actual

    total_predictions = 0



    for watch_id, perf in results['watch_performance'].items():

        # MAPE for this watch

        avg_error = sum(perf['errors']) / len(perf['errors'])

        perf['avg_error_pct'] = round(avg_error, 1)

        perf['accuracy'] = round(100 - avg_error, 1)

        all_errors.extend(perf['errors'])



    results['prediction_accuracy'] = {

        'overall_error_pct': round(sum(all_errors) / len(all_errors), 1),

        'overall_accuracy': round(100 - (sum(all_errors) / len(all_errors)), 1)

    }

        perf['mape'] = round(avg_error, 1)



    # Calculate score (0-100)

    # Based on: accuracy (50%), profit (30%), service level (20%)

    accuracy_score = results['prediction_accuracy']['overall_accuracy'] * 0.5

        # Prediction accuracy: % of predictions within 20% of actual

        accurate_count = sum(1 for err in perf['errors'] if err <= 20)

        perf['accuracy'] = round((accurate_count / len(perf['errors']) * 100), 1)



    # Profit score (normalize to potential max profit)

    potential_profit = results['financial_summary']['total_revenue']  # If perfect predictions

    actual_profit = results['financial_summary']['total_profit']

    profit_score = min(30, (actual_profit / potential_profit * 30)) if potential_profit > 0 else 0

        all_errors.extend(perf['errors'])

        accurate_predictions += accurate_count

        total_predictions += len(perf['errors'])



    # Service level score (based on stockouts)

    total_actual = sum(perf['total_actual'] for perf in results['watch_performance'].values())

    total_sold = sum(perf['total_sold'] for perf in results['watch_performance'].values())

    service_level = (total_sold / total_actual * 100) if total_actual > 0 else 0

    service_score = service_level * 0.2

    # Overall metrics

    overall_mape = sum(all_errors) / len(all_errors)

    overall_accuracy = (accurate_predictions / total_predictions * 100) if total_predictions > 0 else 0



    results['overall_score'] = round(accuracy_score + profit_score + service_score, 1)

    results['prediction_accuracy'] = {

        'mape': round(overall_mape, 1),

        'accuracy': round(overall_accuracy, 1)

    }



    return results

forecast_app/templates/results.html

+19 −12
Original line number Diff line number Diff line
@@ -14,15 +14,22 @@ 


<section class="section">

    <div class="container">

        <!-- Overall Score -->

        <!-- Accuracy Metrics -->

        <div class="columns">

            <div class="column is-6 is-offset-3">

            <div class="column is-6">

                <div class="box has-text-centered" style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white;">

                    <h3 class="title is-3" style="color: white;">Overall Performance Score</h3>

                    <div class="score-display">{{ results.overall_score }}/100</div>

                    <p class="subtitle is-5" style="color: white;">

                        Prediction Accuracy: {{ results.prediction_accuracy.overall_accuracy }}%

                    </p>

                    <h3 class="title is-4" style="color: white;">MAPE</h3>

                    <p class="subtitle is-6" style="color: white; margin-bottom: 0.5rem;">Mean Absolute Percentage Error</p>

                    <div class="score-display">{{ results.prediction_accuracy.mape }}%</div>

                    <p class="subtitle is-6" style="color: white; margin-top: 0.5rem;">Lower is better</p>

                </div>

            </div>

            <div class="column is-6">

                <div class="box has-text-centered" style="background: linear-gradient(135deg, #764ba2 0%, #667eea 100%); color: white;">

                    <h3 class="title is-4" style="color: white;">Prediction Accuracy</h3>

                    <p class="subtitle is-6" style="color: white; margin-bottom: 0.5rem;">% within 20% of actual</p>

                    <div class="score-display">{{ results.prediction_accuracy.accuracy }}%</div>

                    <p class="subtitle is-6" style="color: white; margin-top: 0.5rem;">Higher is better</p>

                </div>

            </div>

        </div>
@@ -195,19 +202,19 @@ 
        <div class="box">

            <h3 class="title is-4">Key Insights</h3>

            <div class="content">

                {% if results.prediction_accuracy.overall_accuracy >= 90 %}

                {% if results.prediction_accuracy.accuracy >= 90 %}

                <div class="notification is-success">

                    <p><strong>Excellent work!</strong> Your predictions were highly accurate ({{ results.prediction_accuracy.overall_accuracy }}%).</p>

                    <p><strong>Excellent work!</strong> Your predictions were highly accurate ({{ results.prediction_accuracy.accuracy }}% within 20% of actual, MAPE: {{ results.prediction_accuracy.mape }}%).</p>

                    <p>You demonstrated strong understanding of demand patterns and seasonality.</p>

                </div>

                {% elif results.prediction_accuracy.overall_accuracy >= 75 %}

                {% elif results.prediction_accuracy.accuracy >= 75 %}

                <div class="notification is-info">

                    <p><strong>Good job!</strong> Your predictions were reasonably accurate ({{ results.prediction_accuracy.overall_accuracy }}%).</p>

                    <p><strong>Good job!</strong> Your predictions were reasonably accurate ({{ results.prediction_accuracy.accuracy }}% within 20% of actual, MAPE: {{ results.prediction_accuracy.mape }}%).</p>

                    <p>There's room for improvement - review the monthly patterns to identify where you can refine your forecasts.</p>

                </div>

                {% else %}

                <div class="notification is-warning">

                    <p><strong>Keep learning!</strong> Your predictions had significant variance from actual demand ({{ results.prediction_accuracy.overall_accuracy }}% accuracy).</p>

                    <p><strong>Keep learning!</strong> Your predictions had significant variance from actual demand ({{ results.prediction_accuracy.accuracy }}% within 20% of actual, MAPE: {{ results.prediction_accuracy.mape }}%).</p>

                    <p>Consider: Did you account for seasonal patterns? Did you follow the growth trends?</p>

                </div>

                {% endif %}