← Mike Higgins Home
Custom Code Experiment •

Wayback Machine

Source Code View HTML / CSS / JavaScript
<!DOCTYPE html>
<html lang="en-GB">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Wayback Machine</title>
  <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/diff-match-patch/1.0.5/diff_match_patch.min.js"></script>
  <style>
    :root {
      --bg-primary: #0a0a0a;
      --bg-secondary: #1a1a1a;
      --bg-surface: #121212;
      --bg-card: #1e1e1e;
      --text-primary: #ffffff;
      --text-secondary: #b0b0b0;
      --text-muted: #888888;
      --accent-primary: #3a86ff;
      --accent-secondary: #8338ec;
      --accent-success: #06d6a0;
      --accent-warning: #ffd166;
      --accent-error: #ef476f;
      --border-color: #333333;
      --border-light: #444444;
      --shadow-color: rgba(0, 0, 0, 0.5);
    }

    * {
      margin: 0;
      padding: 0;
      box-sizing: border-box;
      font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
    }

    body {
      background-color: var(--bg-primary);
      color: var(--text-primary);
      line-height: 1.6;
      min-height: 100vh;
      font-weight: 300;
    }

    .container {
      display: flex;
      min-height: 100vh;
    }

    .sidebar {
      width: 280px;
      background: var(--bg-surface);
      padding: 30px 20px;
      position: fixed;
      height: 100vh;
      overflow-y: auto;
      z-index: 1000;
      transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1);
      border-right: 1px solid var(--border-color);
    }

    .logo-container {
      text-align: center;
      margin-bottom: 40px;
      padding-bottom: 25px;
      border-bottom: 1px solid var(--border-light);
    }

    .logo {
      width: 160px;
      height: auto;
      filter: brightness(0) invert(1) saturate(0) brightness(2);
      margin-bottom: 15px;
      opacity: 0.9;
    }

    .app-title {
      font-size: 1.2rem;
      color: var(--text-primary);
      font-weight: 400;
      letter-spacing: 1px;
      opacity: 0.9;
    }

    .content {
      flex: 1;
      padding: 40px;
      margin-left: 280px;
      max-width: calc(100vw - 280px);
      transition: margin-left 0.3s cubic-bezier(0.4, 0, 0.2, 1);
    }

    header {
      margin-bottom: 50px;
      text-align: left;
    }

    h1 {
      font-size: 2.8rem;
      font-weight: 300;
      margin-bottom: 10px;
      letter-spacing: -0.5px;
      background: linear-gradient(135deg, var(--text-primary), var(--text-secondary));
      -webkit-background-clip: text;
      -webkit-text-fill-color: transparent;
    }

    .subtitle {
      font-size: 1.1rem;
      color: var(--text-secondary);
      font-weight: 300;
      letter-spacing: 0.5px;
    }

    .section {
      margin-bottom: 35px;
      padding-bottom: 25px;
      border-bottom: 1px solid var(--border-color);
    }

    .section h3 {
      font-size: 1rem;
      margin-bottom: 20px;
      color: var(--text-primary);
      font-weight: 500;
      letter-spacing: 1px;
      text-transform: uppercase;
      display: flex;
      align-items: center;
    }

    .section h3::before {
      content: "";
      display: inline-block;
      width: 4px;
      height: 16px;
      background: var(--accent-primary);
      margin-right: 12px;
      border-radius: 2px;
    }

    .radio-group {
      display: flex;
      flex-direction: column;
      gap: 10px;
    }

    .radio-group label {
      display: flex;
      align-items: center;
      cursor: pointer;
      padding: 12px 15px;
      border-radius: 8px;
      transition: all 0.2s ease;
      border: 1px solid transparent;
    }

    .radio-group label:hover {
      background: var(--bg-card);
      border-color: var(--border-light);
    }

    .radio-group input {
      margin-right: 12px;
      width: 18px;
      height: 18px;
      accent-color: var(--accent-primary);
      background: transparent;
    }

    .slider-container {
      margin-top: 20px;
    }

    .slider-container label {
      display: flex;
      justify-content: space-between;
      align-items: center;
      margin-bottom: 12px;
      font-weight: 400;
      color: var(--text-primary);
    }

    input[type="range"] {
      width: 100%;
      height: 4px;
      border-radius: 2px;
      background: var(--border-light);
      outline: none;
      -webkit-appearance: none;
    }

    input[type="range"]::-webkit-slider-thumb {
      -webkit-appearance: none;
      width: 20px;
      height: 20px;
      border-radius: 50%;
      background: var(--text-primary);
      cursor: pointer;
      border: 3px solid var(--bg-surface);
      box-shadow: 0 2px 8px var(--shadow-color);
      transition: all 0.2s ease;
    }

    input[type="range"]::-webkit-slider-thumb:hover {
      transform: scale(1.1);
      box-shadow: 0 4px 12px var(--shadow-color);
    }

    .slider-value {
      font-weight: 500;
      color: var(--text-primary);
      background: var(--bg-card);
      padding: 4px 12px;
      border-radius: 20px;
      font-size: 0.9rem;
      min-width: 40px;
      text-align: center;
    }

    .info-box {
      background: var(--bg-card);
      padding: 20px;
      border-radius: 12px;
      font-size: 0.9rem;
      border: 1px solid var(--border-color);
      color: var(--text-secondary);
    }

    .info-box ol {
      padding-left: 20px;
      margin-top: 12px;
    }

    .info-box li {
      margin-bottom: 8px;
      line-height: 1.5;
    }

    form {
      display: flex;
      gap: 15px;
      margin-bottom: 40px;
      flex-wrap: wrap;
      background: var(--bg-surface);
      padding: 30px;
      border-radius: 16px;
      border: 1px solid var(--border-color);
    }

    .input-group {
      flex: 1;
      min-width: 300px;
    }

    input[type="text"] {
      width: 100%;
      padding: 16px 20px;
      background: var(--bg-card);
      border: 1px solid var(--border-light);
      border-radius: 10px;
      font-size: 16px;
      transition: all 0.2s ease;
      color: var(--text-primary);
    }

    input[type="text"]:focus {
      border-color: var(--accent-primary);
      outline: none;
      box-shadow: 0 0 0 2px rgba(58, 134, 255, 0.1);
    }

    button {
      padding: 16px 35px;
      background: var(--bg-card);
      color: var(--text-primary);
      border: 1px solid var(--border-light);
      border-radius: 10px;
      cursor: pointer;
      font-size: 16px;
      font-weight: 500;
      transition: all 0.2s ease;
      min-width: 140px;
    }

    button:hover {
      background: var(--accent-primary);
      border-color: var(--accent-primary);
      transform: translateY(-1px);
    }

    button:active {
      transform: translateY(0);
    }

    button:disabled {
      opacity: 0.5;
      cursor: not-allowed;
      transform: none;
    }

    .primary-button {
      background: var(--accent-primary);
      border-color: var(--accent-primary);
    }

    .chart-container {
      background: var(--bg-surface);
      border-radius: 16px;
      padding: 25px;
      border: 1px solid var(--border-color);
      margin-bottom: 30px;
      position: relative;
    }

    .chart-container::before {
      content: "";
      position: absolute;
      top: 0;
      left: 0;
      right: 0;
      height: 1px;
      background: linear-gradient(90deg, transparent, var(--accent-primary), transparent);
    }

    .hidden {
      display: none;
    }

    .alert {
      padding: 16px 20px;
      border-radius: 10px;
      margin-bottom: 20px;
      font-weight: 400;
      border: 1px solid;
      backdrop-filter: blur(10px);
    }

    .alert-success {
      background: rgba(6, 214, 160, 0.1);
      border-color: var(--accent-success);
      color: var(--accent-success);
    }

    .alert-warning {
      background: rgba(255, 209, 102, 0.1);
      border-color: var(--accent-warning);
      color: var(--accent-warning);
    }

    .alert-error {
      background: rgba(239, 71, 111, 0.1);
      border-color: var(--accent-error);
      color: var(--accent-error);
    }

    .alert-info {
      background: rgba(58, 134, 255, 0.1);
      border-color: var(--accent-primary);
      color: var(--accent-primary);
    }

    .loading {
      display: inline-block;
      width: 20px;
      height: 20px;
      border: 2px solid rgba(255, 255, 255, 0.2);
      border-radius: 50%;
      border-top-color: var(--text-primary);
      animation: spin 1s ease-in-out infinite;
      margin-right: 10px;
    }

    @keyframes spin {
      to {
        transform: rotate(360deg);
      }
    }

    .progress-container {
      margin: 20px 0;
      background: var(--bg-surface);
      padding: 25px;
      border-radius: 12px;
      border: 1px solid var(--border-color);
    }

    .progress-bar {
      height: 6px;
      background-color: var(--border-light);
      border-radius: 3px;
      overflow: hidden;
      margin-bottom: 12px;
    }

    .progress-fill {
      height: 100%;
      background: linear-gradient(90deg, var(--accent-primary), var(--accent-secondary));
      width: 0%;
      transition: width 0.5s cubic-bezier(0.4, 0, 0.2, 1);
      border-radius: 3px;
    }

    .page-list {
      max-height: 400px;
      overflow-y: auto;
      border: 1px solid var(--border-color);
      border-radius: 10px;
      background: var(--bg-surface);
    }

    .page-item {
      padding: 18px 20px;
      border-bottom: 1px solid var(--border-color);
      cursor: pointer;
      transition: all 0.2s ease;
    }

    .page-item:hover {
      background: var(--bg-card);
    }

    .page-item:last-child {
      border-bottom: none;
    }

    .page-url {
      font-weight: 400;
      margin-bottom: 8px;
      color: var(--text-primary);
      font-size: 0.95rem;
    }

    .page-changes {
      color: var(--accent-primary);
      font-size: 0.85rem;
      font-weight: 500;
    }

    .expander {
      margin-top: 15px;
      border: 1px solid var(--border-color);
      border-radius: 10px;
      overflow: hidden;
      background: var(--bg-surface);
    }

    .expander-header {
      padding: 18px 20px;
      background: var(--bg-card);
      cursor: pointer;
      display: flex;
      justify-content: space-between;
      align-items: center;
      font-weight: 500;
      color: var(--text-primary);
      transition: background 0.2s ease;
    }

    .expander-header:hover {
      background: var(--bg-secondary);
    }

    .expander-content {
      padding: 20px;
      background: var(--bg-surface);
      border-top: 1px solid var(--border-color);
    }

    .columns {
      display: flex;
      gap: 20px;
    }

    .column {
      flex: 1;
    }

    select {
      width: 100%;
      padding: 14px 15px;
      background: var(--bg-card);
      border: 1px solid var(--border-light);
      border-radius: 8px;
      font-size: 15px;
      margin-bottom: 15px;
      color: var(--text-primary);
      font-weight: 400;
    }

    pre {
      background: var(--bg-card);
      padding: 20px;
      border-radius: 8px;
      overflow-x: auto;
      font-size: 0.85rem;
      border: 1px solid var(--border-color);
      max-height: 300px;
      overflow-y: auto;
      color: var(--text-secondary);
      font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
    }

    .diff-added {
      background: rgba(6, 214, 160, 0.2);
      color: var(--accent-success);
      padding: 2px 4px;
      border-radius: 3px;
      border-left: 2px solid var(--accent-success);
    }

    .diff-removed {
      background: rgba(239, 71, 111, 0.2);
      color: var(--accent-error);
      padding: 2px 4px;
      border-radius: 3px;
      border-left: 2px solid var(--accent-error);
    }

    .filter-options {
      margin-bottom: 25px;
    }

    .filter-options label {
      display: block;
      margin-bottom: 10px;
      cursor: pointer;
      padding: 12px 15px;
      border-radius: 8px;
      transition: all 0.2s ease;
      border: 1px solid transparent;
      background: var(--bg-card);
    }

    .filter-options label:hover {
      border-color: var(--border-light);
      background: var(--bg-surface);
    }

    .download-section {
      background: var(--bg-surface);
      border-radius: 16px;
      padding: 30px;
      border: 1px solid var(--border-color);
    }

    .section-header {
      font-size: 1.6rem;
      margin: 40px 0 30px 0;
      padding-bottom: 15px;
      border-bottom: 1px solid var(--border-color);
      color: var(--text-primary);
      font-weight: 300;
      letter-spacing: -0.3px;
    }

    .results-section {
      margin-top: 30px;
    }

    .mobile-menu-btn {
      display: none;
      position: fixed;
      top: 20px;
      left: 20px;
      z-index: 1001;
      background: var(--bg-card);
      color: var(--text-primary);
      border: 1px solid var(--border-color);
      border-radius: 8px;
      padding: 12px;
      cursor: pointer;
      transition: all 0.2s ease;
      width: 44px;
      height: 44px;
      align-items: center;
      justify-content: center;
    }

    .mobile-menu-btn:hover {
      background: var(--bg-surface);
      border-color: var(--accent-primary);
    }

    .mobile-overlay {
      display: none;
      position: fixed;
      top: 0;
      left: 0;
      right: 0;
      bottom: 0;
      background: rgba(0, 0, 0, 0.8);
      z-index: 999;
      backdrop-filter: blur(4px);
    }

    @media (max-width: 1024px) {
      .sidebar {
        transform: translateX(-100%);
        width: 320px;
        background: var(--bg-surface);
      }

      .sidebar.active {
        transform: translateX(0);
        box-shadow: 0 0 50px var(--shadow-color);
      }

      .content {
        margin-left: 0;
        max-width: 100%;
        padding: 80px 25px 25px;
      }

      .mobile-menu-btn {
        display: flex;
      }

      .mobile-overlay.active {
        display: block;
      }

      .columns {
        flex-direction: column;
      }

      form {
        flex-direction: column;
        padding: 25px;
      }

      .input-group {
        min-width: 100%;
      }

      h1 {
        font-size: 2.2rem;
      }

      .section-header {
        font-size: 1.4rem;
      }
    }

    @media (max-width: 768px) {
      .sidebar {
        width: 100%;
        max-width: 280px;
      }

      .content {
        padding: 70px 20px 20px;
      }

      h1 {
        font-size: 2rem;
      }

      .subtitle {
        font-size: 1rem;
      }

      form {
        padding: 20px;
      }

      .chart-container {
        padding: 20px;
      }

      button {
        width: 100%;
      }
    }

    @media (max-width: 480px) {
      h1 {
        font-size: 1.8rem;
      }

      .content {
        padding: 70px 15px 15px;
      }

      form {
        padding: 20px 15px;
      }

      .chart-container {
        padding: 15px;
      }

      .section-header {
        font-size: 1.3rem;
      }
    }

    ::-webkit-scrollbar {
      width: 8px;
      height: 8px;
    }

    ::-webkit-scrollbar-track {
      background: var(--bg-surface);
      border-radius: 4px;
    }

    ::-webkit-scrollbar-thumb {
      background: var(--border-light);
      border-radius: 4px;
    }

    ::-webkit-scrollbar-thumb:hover {
      background: var(--text-muted);
    }

    .js-plotly-plot .plotly .modebar {
      background: transparent !important;
    }

    .js-plotly-plot .plotly .modebar-btn path {
      fill: var(--text-secondary) !important;
    }

    .js-plotly-plot .plotly .main-svg {
      background: transparent !important;
    }
  </style>
</head>

<body>
  <button class="mobile-menu-btn" id="mobileMenuBtn">
    <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
      <path d="M3 12h18M3 6h18M3 18h18"/>
    </svg>
  </button>
  <div class="mobile-overlay" id="mobileOverlay"></div>

  <div class="container">
    <div class="sidebar" id="sidebar">
      <div class="logo-container">
        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/0/01/Wayback_Machine_logo_2010.svg/1200px-Wayback_Machine_logo_2010.svg.png" alt="Wayback Machine" class="logo">
        <div class="app-title">ARCHIVE ANALYSER</div>
      </div>

      <div class="section">
        <h3>Visualisation Options</h3>
        <div class="radio-group">
          <label><input type="radio" name="visType" value="line" checked> Stacked Line Chart</label>
          <label><input type="radio" name="visType" value="bar"> Stacked Bar Chart</label>
        </div>

        <div class="slider-container">
          <label>Top folders to display: <span id="folderCountValue" class="slider-value">10</span></label>
          <input type="range" id="folderCount" min="5" max="50" value="10">
        </div>
      </div>

      <div class="section">
        <h3>About This Tool</h3>
        <div class="info-box">
          Analyse website evolution through Wayback Machine archives:
          <ol>
            <li>URL Retrieval from Internet Archive</li>
            <li>Folder Structure Evolution</li>
            <li>HTTP Status Code Analysis</li>
            <li>Frequently Changed Pages</li>
            <li>robots.txt Change Tracking</li>
          </ol>
        </div>
      </div>
    </div>

    <div class="content">
      <header>
        <h1>Wayback Machine Analyser</h1>
        <div class="subtitle">Explore website evolution through historical archives</div>
      </header>

      <form id="domainForm">
        <div class="input-group">
          <input type="text" id="domainInput" placeholder="Enter domain (e.g., example.com)" required>
        </div>
        <button type="submit" id="submitButton" class="primary-button">Analyse Website</button>
      </form>

      <div id="alerts"></div>
      <div id="progressContainer" class="progress-container hidden">
        <div class="progress-bar">
          <div class="progress-fill" id="progressFill"></div>
        </div>
        <div id="progressText">Processing...</div>
      </div>

      <div id="results" class="hidden">
        <div class="results-section">
          <h2 class="section-header">Analysis Results</h2>

          <div class="section">
            <h3>Folder Structure Evolution</h3>
            <div class="chart-container">
              <div id="folderViz"></div>
            </div>
          </div>

          <div class="section">
            <h3>Status Code Evolution</h3>
            <div class="chart-container">
              <div id="statusViz"></div>
            </div>
          </div>

          <div class="section">
            <h3>Frequently Changed Pages</h3>
            <div class="chart-container">
              <div id="changesViz"></div>
            </div>
            <div id="pageListContainer"></div>
            <div id="pageDetails" class="hidden">
              <h4>Change History</h4>
              <div id="pageHistory"></div>
            </div>
          </div>

          <div class="section">
            <h3>robots.txt Changes</h3>
            <div class="chart-container">
              <div id="robotsViz"></div>
            </div>
            <div id="robotsComparison">
              <h4>Compare robots.txt Versions</h4>
              <div class="columns">
                <div class="column">
                  <select id="version1"></select>
                  <pre id="content1"></pre>
                </div>
                <div class="column">
                  <select id="version2"></select>
                  <pre id="content2"></pre>
                </div>
              </div>
              <div id="diffResult"></div>
            </div>
          </div>

          <div class="section">
            <h3>Download Data</h3>
            <div class="download-section">
              <div class="filter-options">
                <h4>Filter Options</h4>
                <label><input type="radio" name="filterType" value="all" checked> All (HTML, Images, CSS, JS, etc.)</label>
                <label><input type="radio" name="filterType" value="html"> HTML only</label>
                <label><input type="radio" name="filterType" value="html-images"> HTML + Images</label>
                <label><input type="checkbox" id="uniqueOnly"> Export only unique URLs</label>
              </div>
              <button id="downloadButton">Download Filtered URLs</button>
              <div id="downloadInfo"></div>
            </div>
          </div>
        </div>
      </div>
    </div>
  </div>

  <script>
    class WaybackAnalyser {
      constructor() {
        this.uniqueUrls = [];
        this.domain = '';
        this.visType = 'line';
        this.topFoldersCount = 10;
        this.robotsData = [];
        this.initializeEventListeners();
        this.setupMobileMenu();
        this.setupPlotlyTheme();
      }

      setupPlotlyTheme() {
        // Set Plotly to use dark theme
        const darkTheme = {
          paper_bgcolor: 'transparent',
          plot_bgcolor: 'transparent',
          font: { color: '#ffffff' },
          xaxis: {
            gridcolor: 'rgba(255,255,255,0.1)',
            linecolor: 'rgba(255,255,255,0.2)',
            zerolinecolor: 'rgba(255,255,255,0.1)',
          },
          yaxis: {
            gridcolor: 'rgba(255,255,255,0.1)',
            linecolor: 'rgba(255,255,255,0.2)',
            zerolinecolor: 'rgba(255,255,255,0.1)',
          },
          legend: {
            font: { color: '#ffffff' }
          }
        };
        Plotly.setPlotConfig({
          modeBarButtonsToRemove: ['sendDataToCloud'],
          displaylogo: false,
          plotGlPixelRatio: 2
        });
      }

      setupMobileMenu() {
        const mobileMenuBtn = document.getElementById('mobileMenuBtn');
        const sidebar = document.getElementById('sidebar');
        const overlay = document.getElementById('mobileOverlay');

        const toggleMenu = () => {
          sidebar.classList.toggle('active');
          overlay.classList.toggle('active');
          document.body.style.overflow = sidebar.classList.contains('active') ? 'hidden' : '';
        };

        mobileMenuBtn.addEventListener('click', toggleMenu);
        overlay.addEventListener('click', toggleMenu);

        sidebar.addEventListener('click', (e) => {
          if (window.innerWidth <= 1024) {
            toggleMenu();
          }
        });
      }

      initializeEventListeners() {
        document.getElementById('domainForm').addEventListener('submit', (e) => {
          e.preventDefault();
          this.fetchUrls();
        });

        document.querySelectorAll('input[name="visType"]').forEach(radio => {
          radio.addEventListener('change', (e) => {
            this.visType = e.target.value;
            this.updateVisualisations();
          });
        });

        document.getElementById('folderCount').addEventListener('input', (e) => {
          this.topFoldersCount = e.target.value;
          document.getElementById('folderCountValue').textContent = e.target.value;
          this.updateVisualisations();
        });

        document.getElementById('downloadButton').addEventListener('click', () => {
          this.downloadUrls();
        });
      }

      async fetchUrls() {
        const domain = document.getElementById('domainInput').value.trim();
        if (!domain) {
          this.showAlert('Please enter a domain', 'warning');
          return;
        }

        this.domain = domain;
        this.showProgress(0, 'Fetching URLs... This may take a while for large websites.');
        document.getElementById('submitButton').disabled = true;
        document.getElementById('submitButton').textContent = 'Processing...';

        try {
          const urls = await this.getUniqueUrls(domain);
          this.uniqueUrls = urls;

          if (urls.length > 0) {
            this.showAlert(`Found ${urls.length} URLs`, 'success');
            document.getElementById('results').classList.remove('hidden');
            this.updateAllVisualisations();
            document.getElementById('results').scrollIntoView({
              behavior: 'smooth',
              block: 'start'
            });
          } else {
            this.showAlert('No URLs found. Check the domain name.', 'warning');
          }
        } catch (error) {
          this.showAlert(`Error fetching URLs: ${error.message}`, 'error');
        } finally {
          this.hideProgress();
          document.getElementById('submitButton').disabled = false;
          document.getElementById('submitButton').textContent = 'Analyse Website';
        }
      }

      async getUniqueUrls(domain, retries = 3) {
        domain = domain.replace(/^https?:\/\//, '').replace(/\/$/, '');

        const proxyUrls = [
          'https://corsproxy.io/?',
          'https://api.allorigins.win/raw?url=',
          'https://cors-anywhere.herokuapp.com/'
        ];

        const baseUrl = "http://web.archive.org/cdx/search/cdx";
        const params = new URLSearchParams({
          url: `${domain}/*`,
          output: "json",
          fl: "original,timestamp,statuscode,digest",
          pageSize: "500"
        });

        let lastError = null;

        for (let attempt = 0; attempt < retries; attempt++) {
          for (const proxyUrl of proxyUrls) {
            try {
              const fullUrl = `${baseUrl}?${params}`;
              let apiUrl;

              if (proxyUrl.includes('corsproxy.io')) {
                apiUrl = `${proxyUrl}${encodeURIComponent(fullUrl)}`;
              } else if (proxyUrl.includes('allorigins.win')) {
                apiUrl = `${proxyUrl}${encodeURIComponent(fullUrl)}`;
              } else {
                apiUrl = `${proxyUrl}${fullUrl}`;
              }

              this.showProgress(10, `Attempt ${attempt + 1}: Fetching data...`);
              const response = await fetch(apiUrl, {
                method: 'GET',
                headers: { 'Accept': 'application/json' },
                mode: 'cors'
              });

              if (!response.ok) continue;

              const data = await response.json();
              if (!data || data.length <= 1) return [];

              const uniqueUrls = [];
              for (let i = 1; i < data.length; i++) {
                const item = data[i];
                const url = this.cleanUrl(item[0]);
                uniqueUrls.push({
                  url: url,
                  timestamp: item[1],
                  statuscode: item[2],
                  digest: item[3]
                });

                if (i % 50 === 0) {
                  const progress = 10 + (i / data.length) * 80;
                  this.showProgress(progress, `Processing ${i} of ${data.length} URLs...`);
                }
              }

              this.showProgress(100, 'Processing complete!');
              return uniqueUrls;
            } catch (error) {
              lastError = error;
              await this.sleep(1000);
            }
          }
        }

        throw new Error(`All attempts failed. Last error: ${lastError?.message || 'Unknown error'}`);
      }

      async sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
      }

      cleanUrl(url) {
        try {
          if (!url.startsWith('http')) url = 'http://' + url;
          const urlObj = new URL(url);
          return `${urlObj.protocol}//${urlObj.hostname}${urlObj.pathname}${urlObj.search}${urlObj.hash}`;
        } catch (e) {
          return url.replace(/\/\/.*@/, '//').replace(/^\/\//, 'http://');
        }
      }

      getTopFolder(url) {
        try {
          const urlObj = new URL(url);
          const path = urlObj.pathname.replace(/^\/|\/$/g, '');
          if (!path) return "Root";
          return `/${path.split('/')[0]}/`;
        } catch (e) {
          return "Invalid URL";
        }
      }

      groupStatusCode(code) {
        try {
          const numCode = parseInt(code);
          if (numCode < 200) return '1xx';
          if (numCode < 300) return '2xx';
          if (numCode < 400) return '3xx';
          if (numCode < 500) return '4xx';
          return '5xx';
        } catch {
          return 'Unknown';
        }
      }

      updateAllVisualisations() {
        this.visualiseFolderTypesOverTime();
        this.visualiseStatusCodesOverTime();
        this.visualiseTopChangingPages();
        this.visualiseRobotsChanges();
      }

      updateVisualisations() {
        if (this.uniqueUrls.length > 0) {
          this.updateAllVisualisations();
        }
      }

      visualiseFolderTypesOverTime() {
        if (this.uniqueUrls.length === 0) return;

        const data = {};
        this.uniqueUrls.forEach(item => {
          const year = item.timestamp.substring(0, 4);
          const folder = this.getTopFolder(item.url);
          if (!data[year]) data[year] = {};
          if (!data[year][folder]) data[year][folder] = 0;
          data[year][folder]++;
        });

        const years = Object.keys(data).sort();
        const allFolders = new Set();
        years.forEach(year => Object.keys(data[year]).forEach(folder => allFolders.add(folder)));

        const folderTotals = {};
        allFolders.forEach(folder => {
          folderTotals[folder] = years.reduce((sum, year) => sum + (data[year][folder] || 0), 0);
        });

        const sortedFolders = Object.keys(folderTotals).sort((a, b) => folderTotals[b] - folderTotals[a]);
        const topFolders = sortedFolders.slice(0, this.topFoldersCount);

        const traces = topFolders.map(folder => ({
          x: years,
          y: years.map(year => data[year][folder] || 0),
          type: this.visType === 'line' ? 'scatter' : 'bar',
          mode: this.visType === 'line' ? 'lines' : null,
          stackgroup: this.visType === 'line' ? 'one' : null,
          name: folder,
          marker: { color: '#3a86ff' }
        }));

        const layout = {
          title: 'Evolution of Website Structure Over Time',
          xaxis: { title: 'Year', type: 'category' },
          yaxis: { title: 'Number of URLs' },
          barmode: this.visType === 'bar' ? 'stack' : null,
          height: 500,
          font: { color: '#ffffff' },
          paper_bgcolor: 'transparent',
          plot_bgcolor: 'transparent',
          legend: { font: { color: '#ffffff' } }
        };

        Plotly.newPlot('folderViz', traces, layout);
      }

      visualiseStatusCodesOverTime() {
        if (this.uniqueUrls.length === 0) return;

        const data = {};
        this.uniqueUrls.forEach(item => {
          const year = item.timestamp.substring(0, 4);
          const statusGroup = this.groupStatusCode(item.statuscode);
          if (!data[year]) data[year] = {};
          if (!data[year][statusGroup]) data[year][statusGroup] = 0;
          data[year][statusGroup]++;
        });

        const years = Object.keys(data).sort();
        const statusGroups = ['1xx', '2xx', '3xx', '4xx', '5xx', 'Unknown'];
        const colors = ['#3a86ff', '#06d6a0', '#ffd166', '#ff9e00', '#ef476f', '#888888'];

        const traces = statusGroups.map((group, index) => ({
          x: years,
          y: years.map(year => data[year][group] || 0),
          type: this.visType === 'line' ? 'scatter' : 'bar',
          mode: this.visType === 'line' ? 'lines' : null,
          stackgroup: this.visType === 'line' ? 'one' : null,
          name: group,
          marker: { color: colors[index] }
        }));

        const layout = {
          title: 'Evolution of Status Codes Over Time',
          xaxis: { title: 'Year', type: 'category' },
          yaxis: { title: 'Number of URLs' },
          barmode: this.visType === 'bar' ? 'stack' : null,
          height: 500,
          font: { color: '#ffffff' },
          paper_bgcolor: 'transparent',
          plot_bgcolor: 'transparent',
          legend: { font: { color: '#ffffff' } }
        };

        Plotly.newPlot('statusViz', traces, layout);
      }

      getTopChangingPages() {
        const pageChanges = {};
        this.uniqueUrls.forEach(item => {
          if (!pageChanges[item.url]) pageChanges[item.url] = new Set();
          pageChanges[item.url].add(item.digest);
        });

        return Object.keys(pageChanges)
          .map(url => ({ url, changes: pageChanges[url].size }))
          .sort((a, b) => b.changes - a.changes)
          .slice(0, this.topFoldersCount);
      }

      visualiseTopChangingPages() {
        if (this.uniqueUrls.length === 0) return;

        const topChangingPages = this.getTopChangingPages();
        const trace = {
          x: topChangingPages.map(item => item.changes),
          y: topChangingPages.map(item => {
            const maxLength = 60;
            return item.url.length > maxLength ?
              item.url.substring(0, maxLength) + '...' :
              item.url;
          }),
          type: 'bar',
          orientation: 'h',
          marker: { color: '#3a86ff' }
        };

        const layout = {
          title: `Top ${this.topFoldersCount} Frequently Changed Pages`,
          xaxis: { title: 'Number of Changes' },
          yaxis: { title: 'Page', automargin: true },
          height: Math.max(400, topChangingPages.length * 30),
          margin: { l: 200 },
          font: { color: '#ffffff' },
          paper_bgcolor: 'transparent',
          plot_bgcolor: 'transparent'
        };

        Plotly.newPlot('changesViz', [trace], layout);
        this.createPageList(topChangingPages);
      }

      createPageList(pages) {
        const pageList = document.createElement('div');
        pageList.className = 'page-list';

        pages.forEach(page => {
          const pageItem = document.createElement('div');
          pageItem.className = 'page-item';
          pageItem.innerHTML = `
            <div class="page-url">${page.url}</div>
            <div class="page-changes">Changes: ${page.changes}</div>
          `;
          pageItem.addEventListener('click', () => this.showPageHistory(page.url));
          pageList.appendChild(pageItem);
        });

        document.getElementById('pageListContainer').innerHTML = '<h4>Click on a page to view its change history:</h4>';
        document.getElementById('pageListContainer').appendChild(pageList);
      }

      showPageHistory(pageUrl) {
        const pageUrls = this.uniqueUrls
          .filter(item => item.url === pageUrl)
          .sort((a, b) => b.timestamp.localeCompare(a.timestamp))
          .slice(0, 50);

        let historyHTML = '';
        pageUrls.forEach((item, index) => {
          const date = this.formatTimestamp(item.timestamp);
          historyHTML += `
            <div class="expander">
              <div class="expander-header">
                <span>${date}</span>
                <span>Status: ${item.statuscode}</span>
              </div>
              <div class="expander-content">
                <p><strong>URL:</strong> ${item.url}</p>
                <p><strong>Digest:</strong> ${item.digest}</p>
                <a href="https://web.archive.org/web/${item.timestamp}/${item.url}" target="_blank" style="color: #3a86ff;">View Version</a>
              </div>
            </div>
          `;
        });

        document.getElementById('pageHistory').innerHTML = historyHTML;
        document.getElementById('pageDetails').classList.remove('hidden');

        document.querySelectorAll('.expander-header').forEach(header => {
          header.addEventListener('click', () => {
            const content = header.nextElementSibling;
            content.style.display = content.style.display === 'none' ? 'block' : 'none';
          });
        });
      }

      async fetchRobotsData() {
        if (this.robotsData.length > 0) return this.robotsData;

        const proxyUrl = 'https://corsproxy.io/?';
        const baseUrl = "http://web.archive.org/cdx/search/cdx";
        const params = new URLSearchParams({
          url: `${this.domain}/robots.txt`,
          output: "json",
          fl: "timestamp,statuscode,digest,original",
          filter: "statuscode:200",
          collapse: "digest"
        });

        try {
          const fullUrl = `${baseUrl}?${params}`;
          const apiUrl = `${proxyUrl}${encodeURIComponent(fullUrl)}`;
          const response = await fetch(apiUrl);

          if (!response.ok) return [];

          const data = await response.json();
          this.robotsData = data && data.length > 1 ? data.slice(1) : [];
          return this.robotsData;
        } catch (error) {
          return [];
        }
      }

      async visualiseRobotsChanges() {
        const robotsData = await this.fetchRobotsData();

        if (robotsData.length === 0) {
          document.getElementById('robotsViz').innerHTML = '<p style="color: #888;">No robots.txt changes found.</p>';
          return;
        }

        const changes = {};
        robotsData.forEach(item => {
          const digest = item[2];
          if (!changes[digest]) {
            changes[digest] = {
              timestamp: item[0],
              digest: digest,
              original: item[3]
            };
          }
        });

        const uniqueChanges = Object.values(changes)
          .sort((a, b) => a.timestamp.localeCompare(b.timestamp));

        const trace = {
          x: uniqueChanges.map(change => this.formatTimestamp(change.timestamp)),
          y: new Array(uniqueChanges.length).fill(1),
          mode: 'markers+lines',
          marker: { size: 12, color: '#3a86ff' },
          line: { color: '#3a86ff', width: 2 }
        };

        const layout = {
          title: 'Timeline of Unique robots.txt Changes',
          xaxis: { title: 'Date' },
          yaxis: { showticklabels: false, showgrid: false },
          height: 300,
          showlegend: false,
          font: { color: '#ffffff' },
          paper_bgcolor: 'transparent',
          plot_bgcolor: 'transparent'
        };

        Plotly.newPlot('robotsViz', [trace], layout);
        this.populateVersionSelectors(uniqueChanges);
      }

      populateVersionSelectors(changes) {
        const version1 = document.getElementById('version1');
        const version2 = document.getElementById('version2');

        version1.innerHTML = '';
        version2.innerHTML = '';

        changes.forEach((change, index) => {
          const option1 = document.createElement('option');
          const option2 = document.createElement('option');
          const date = this.formatTimestamp(change.timestamp);

          option1.textContent = option2.textContent = date;
          option1.value = option2.value = index;
          version1.appendChild(option1);
          version2.appendChild(option2);
        });

        if (changes.length > 1) version2.value = 1;

        version1.addEventListener('change', () => this.compareRobotsVersions());
        version2.addEventListener('change', () => this.compareRobotsVersions());
        this.compareRobotsVersions();
      }

      async compareRobotsVersions() {
        const version1Index = document.getElementById('version1').value;
        const version2Index = document.getElementById('version2').value;

        if (!version1Index || !version2Index) return;

        const changes = Object.values(this.robotsData.reduce((acc, item) => {
          const digest = item[2];
          if (!acc[digest]) {
            acc[digest] = {
              timestamp: item[0],
              digest: digest,
              original: item[3]
            };
          }
          return acc;
        }, {})).sort((a, b) => a.timestamp.localeCompare(b.timestamp));

        const version1 = changes[version1Index];
        const version2 = changes[version2Index];

        try {
          const content1 = await this.fetchRobotsContent(version1.timestamp);
          const content2 = await this.fetchRobotsContent(version2.timestamp);

          document.getElementById('content1').textContent = content1 || 'Content not available';
          document.getElementById('content2').textContent = content2 || 'Content not available';

          if (content1 && content2) {
            this.showDiff(content1, content2);
          }
        } catch (error) {
          console.warn('Error comparing robots versions:', error);
        }
      }

      async fetchRobotsContent(timestamp) {
        const proxyUrl = 'https://corsproxy.io/?';
        const url = `https://web.archive.org/web/${timestamp}id_/${this.domain}/robots.txt`;

        try {
          const response = await fetch(`${proxyUrl}${encodeURIComponent(url)}`);
          return response.ok ? await response.text() : null;
        } catch (error) {
          return null;
        }
      }

      showDiff(text1, text2) {
        const dmp = new diff_match_patch();
        const diffs = dmp.diff_main(text1, text2);
        dmp.diff_cleanupSemantic(diffs);

        let diffHTML = '<h4>Differences:</h4><pre>';
        diffs.forEach(([operation, text]) => {
          const lines = text.split('\n');
          lines.forEach(line => {
            if (line.trim() === '') return;
            if (operation === 1) {
              diffHTML += `<div class="diff-added">+ ${line}</div>`;
            } else if (operation === -1) {
              diffHTML += `<div class="diff-removed">- ${line}</div>`;
            } else {
              diffHTML += `<div>  ${line}</div>`;
            }
          });
        });
        diffHTML += '</pre>';
        document.getElementById('diffResult').innerHTML = diffHTML;
      }

      downloadUrls() {
        const filterType = document.querySelector('input[name="filterType"]:checked').value;
        const uniqueOnly = document.getElementById('uniqueOnly').checked;

        let filteredUrls = this.uniqueUrls.filter(item => this.applyFilter(item.url, filterType));

        if (uniqueOnly) {
          const uniqueUrls = {};
          filteredUrls.forEach(item => {
            if (!uniqueUrls[item.url]) uniqueUrls[item.url] = item;
          });
          filteredUrls = Object.values(uniqueUrls);
        }

        let csvContent = "URL,Timestamp,Status Code,Digest\n";
        filteredUrls.forEach(item => {
          csvContent += `"${item.url}","${item.timestamp}","${item.statuscode}","${item.digest}"\n`;
        });

        const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' });
        const link = document.createElement('a');
        const url = URL.createObjectURL(blob);

        link.setAttribute('href', url);
        link.setAttribute('download', `${this.domain}_urls.csv`);
        link.style.visibility = 'hidden';
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);

        document.getElementById('downloadInfo').innerHTML =
          `<div class="alert alert-success">Downloaded ${filteredUrls.length} URLs</div>`;
      }

      applyFilter(url, filterType) {
        if (filterType === 'all') return true;
        if (filterType === 'html') {
          return url.endsWith('.html') || url.endsWith('.htm') || url.endsWith('/') || url.includes('robots.txt');
        }
        if (filterType === 'html-images') {
          return url.endsWith('.html') || url.endsWith('.htm') || url.endsWith('/') ||
                 url.includes('robots.txt') ||
                 url.match(/\.(jpg|jpeg|png|gif|svg|webp)$/i);
        }
        return true;
      }

      formatTimestamp(timestamp) {
        const year = timestamp.substring(0, 4);
        const month = timestamp.substring(4, 6);
        const day = timestamp.substring(6, 8);
        const hour = timestamp.substring(8, 10);
        const minute = timestamp.substring(10, 12);
        const second = timestamp.substring(12, 14);
        return `${year}-${month}-${day} ${hour}:${minute}:${second}`;
      }

      showAlert(message, type) {
        const alertsDiv = document.getElementById('alerts');
        const alert = document.createElement('div');
        alert.className = `alert alert-${type}`;
        alert.textContent = message;
        alertsDiv.appendChild(alert);

        setTimeout(() => alert.remove(), 5000);
      }

      showProgress(percent, message) {
        document.getElementById('progressContainer').classList.remove('hidden');
        document.getElementById('progressFill').style.width = `${percent}%`;
        document.getElementById('progressText').textContent = message;
      }

      hideProgress() {
        document.getElementById('progressContainer').classList.add('hidden');
      }
    }

    document.addEventListener('DOMContentLoaded', () => {
      new WaybackAnalyser();
    });
  </script>
</body>
</html>