diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev index 2c33440d4e2..3356f1f0bca 100644 --- a/docker/.ngc_version.dev +++ b/docker/.ngc_version.dev @@ -1 +1 @@ -nvcr.io/nvidia/pytorch:26.02-py3 \ No newline at end of file +nvcr.io/nvidia/pytorch:26.04-py3 diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json index 075cd7259e4..0608eabcf09 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93343, - "2": 10.92669, - "3": 10.92551, - "4": 10.92532, + "1": 10.93336, + "2": 10.92666, + "3": 10.9255, + "4": 10.92529, "5": 10.93754, - "6": 10.92826, - "7": 10.92387, - "8": 10.92373, - "9": 10.92337, + "6": 10.9282, + "7": 10.9239, + "8": 10.92371, + "9": 10.92334, "10": 10.92739, - "11": 10.9111, - "12": 10.92744, - "13": 10.89422, + "11": 10.91111, + "12": 10.92741, + "13": 10.8942, "14": 10.89282, - "15": 10.8915, - "16": 10.87288, - "17": 10.86943, - "18": 10.86805, - "19": 10.8535, - "20": 10.80094, - "21": 10.78481, - "22": 10.7646, - "23": 10.76759, - "24": 10.75104, - "25": 10.74834, - "26": 10.71952, - "27": 10.68558, - "28": 10.61472, - "29": 10.58963, - "30": 10.55446, - "31": 10.56398, - "32": 10.54721, - "33": 10.51341, - "34": 10.48397, - "35": 10.48648, - "36": 10.46526, - "37": 10.42756, - "38": 10.42669, - "39": 10.39364, - "40": 10.37201, - "41": 10.35126, - "42": 10.34669, - "43": 10.32133, - "44": 10.30202, + "15": 10.89147, + "16": 10.87299, + "17": 10.86945, + "18": 10.86815, + "19": 10.85351, + "20": 10.80099, + "21": 10.7848, + "22": 10.76461, + "23": 10.76766, + "24": 10.75107, + "25": 10.74838, + "26": 10.71962, + "27": 10.68563, + "28": 10.61479, + "29": 10.58967, + "30": 10.55443, + "31": 10.56401, + "32": 10.5473, + "33": 10.51346, + "34": 10.48399, + "35": 10.48652, + "36": 10.4653, + "37": 10.42766, + "38": 10.42672, + "39": 10.39363, + "40": 10.37209, + "41": 10.35129, + "42": 10.34678, + "43": 10.32137, + "44": 10.30203, "45": 10.30545, - "46": 10.26663, - "47": 10.25281, - "48": 10.21608, - "49": 10.20375, - "50": 10.2102 + "46": 10.26667, + "47": 10.25286, + "48": 10.21606, + "49": 10.20378, + "50": 10.21031 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1967.0, - "2": 1681.0, - "3": 1791.0, - "4": 1723.0, - "5": 1710.0, - "6": 1726.0, - "7": 2091.0, - "8": 1799.0, - "9": 1821.0, - "10": 1745.0, - "11": 1701.0, - "12": 1780.0, - "13": 1825.0, - "14": 1958.0, - "15": 1674.0, - "16": 1793.0, - "17": 1823.0, - "18": 1850.0, - "19": 1702.0, - "20": 1636.0, - "21": 1801.0, - "22": 1760.0, - "23": 1782.0, - "24": 1888.0, - "25": 1744.0, - "26": 1797.0, - "27": 1769.0, - "28": 1972.0, - "29": 1927.0, - "30": 1969.0, - "31": 2034.0, - "32": 2050.0, - "33": 2020.0, - "34": 2131.0, - "35": 2040.0, - "36": 2145.0, - "37": 2290.0, - "38": 2234.0, - "39": 2232.0, - "40": 2329.0, - "41": 2441.0, - "42": 2053.0, - "43": 2381.0, - "44": 2323.0, - "45": 2510.0, - "46": 2497.0, - "47": 2456.0, - "48": 2652.0, - "49": 2864.0, - "50": 2585.0 + "1": 1957.0, + "2": 1776.0, + "3": 1717.0, + "4": 1733.0, + "5": 1722.0, + "6": 1746.0, + "7": 2002.0, + "8": 1742.0, + "9": 1845.0, + "10": 1812.0, + "11": 1756.0, + "12": 1686.0, + "13": 1812.0, + "14": 1923.0, + "15": 1687.0, + "16": 1716.0, + "17": 1783.0, + "18": 1842.0, + "19": 1743.0, + "20": 1657.0, + "21": 1818.0, + "22": 1751.0, + "23": 1730.0, + "24": 1859.0, + "25": 1746.0, + "26": 1811.0, + "27": 1731.0, + "28": 1889.0, + "29": 1964.0, + "30": 1951.0, + "31": 2131.0, + "32": 1977.0, + "33": 1959.0, + "34": 2150.0, + "35": 2172.0, + "36": 2077.0, + "37": 2216.0, + "38": 2163.0, + "39": 2182.0, + "40": 2355.0, + "41": 2401.0, + "42": 2022.0, + "43": 2437.0, + "44": 2310.0, + "45": 2449.0, + "46": 2451.0, + "47": 2567.0, + "48": 2642.0, + "49": 2787.0, + "50": 2661.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.89225, - "3": 0.08163, - "4": 0.08422, - "5": 0.08203, - "6": 0.08251, - "7": 0.08165, - "8": 0.08733, - "9": 0.08533, - "10": 0.08163, - "11": 0.08023, - "12": 0.08236, - "13": 0.0813, - "14": 0.08273, - "15": 0.08182, - "16": 0.08156, - "17": 0.08634, - "18": 0.08175, - "19": 0.08144, - "20": 0.08379, - "21": 0.08315, - "22": 0.08162, - "23": 0.08092, - "24": 0.08149, - "25": 0.0819, - "26": 0.08509, - "27": 0.08243, - "28": 0.08027, - "29": 0.08085, - "30": 0.08176, - "31": 0.08133, - "32": 0.08421, - "33": 0.08126, - "34": 0.08115, - "35": 0.08002, - "36": 0.07981, - "37": 0.08035, - "38": 0.08069, - "39": 0.08506, - "40": 0.0867, - "41": 0.08141, - "42": 0.08516, - "43": 0.08312, - "44": 0.0872, - "45": 0.08844, - "46": 0.08728, - "47": 0.08141, - "48": 0.08513, - "49": 0.07969, - "50": 0.08489 + "2": 3.23245, + "3": 0.08958, + "4": 0.08756, + "5": 0.08887, + "6": 0.09427, + "7": 0.08751, + "8": 0.08973, + "9": 0.08693, + "10": 0.08701, + "11": 0.08637, + "12": 0.08945, + "13": 0.08584, + "14": 0.08675, + "15": 0.08667, + "16": 0.08635, + "17": 0.08714, + "18": 0.08606, + "19": 0.08615, + "20": 0.08699, + "21": 0.0873, + "22": 0.08479, + "23": 0.08614, + "24": 0.0869, + "25": 0.08659, + "26": 0.08656, + "27": 0.08562, + "28": 0.08571, + "29": 0.08536, + "30": 0.08532, + "31": 0.08588, + "32": 0.08497, + "33": 0.08598, + "34": 0.08578, + "35": 0.08719, + "36": 0.08782, + "37": 0.0859, + "38": 0.08614, + "39": 0.08519, + "40": 0.08612, + "41": 0.08704, + "42": 0.08591, + "43": 0.0862, + "44": 0.08581, + "45": 0.08542, + "46": 0.08608, + "47": 0.08787, + "48": 0.09011, + "49": 0.08579, + "50": 0.08573 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json index 1b9f7e91548..72e7f7f1b29 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88433, - "2": 10.88455, + "1": 10.88436, + "2": 10.88459, "3": 10.88309, - "4": 10.88497, - "5": 10.87886, - "6": 10.87416, - "7": 10.88029, - "8": 10.87382, - "9": 10.88138, - "10": 10.88114, - "11": 10.87549, - "12": 10.86262, - "13": 10.87418, - "14": 10.86552, - "15": 10.83746, - "16": 10.83285, - "17": 10.83752, + "4": 10.88494, + "5": 10.87889, + "6": 10.87417, + "7": 10.88025, + "8": 10.87377, + "9": 10.88146, + "10": 10.88113, + "11": 10.87554, + "12": 10.86259, + "13": 10.87412, + "14": 10.86553, + "15": 10.83742, + "16": 10.83275, + "17": 10.83755, "18": 10.82158, - "19": 10.83534, - "20": 10.73039, - "21": 10.73845, - "22": 10.72167, - "23": 10.71282, - "24": 10.68134, - "25": 10.67711, - "26": 10.67859, - "27": 10.63636, - "28": 10.57712, - "29": 10.54434, - "30": 10.52198, + "19": 10.8353, + "20": 10.73041, + "21": 10.73852, + "22": 10.72163, + "23": 10.71275, + "24": 10.68131, + "25": 10.67714, + "26": 10.67861, + "27": 10.63632, + "28": 10.57717, + "29": 10.54437, + "30": 10.52205, "31": 10.51451, - "32": 10.49261, - "33": 10.47084, + "32": 10.49257, + "33": 10.47083, "34": 10.43254, - "35": 10.44114, - "36": 10.41271, + "35": 10.4411, + "36": 10.4127, "37": 10.37829, - "38": 10.39605, - "39": 10.35296, - "40": 10.34738, - "41": 10.32554, - "42": 10.29871, - "43": 10.2881, - "44": 10.24608, - "45": 10.27178, - "46": 10.22622, - "47": 10.22132, + "38": 10.39602, + "39": 10.35301, + "40": 10.34743, + "41": 10.32556, + "42": 10.29877, + "43": 10.28817, + "44": 10.2461, + "45": 10.27179, + "46": 10.22628, + "47": 10.22134, "48": 10.17358, - "49": 10.17262, - "50": 10.17363 + "49": 10.17266, + "50": 10.17368 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1762.0, - "2": 1751.0, - "3": 1680.0, - "4": 1694.0, - "5": 1780.0, - "6": 1620.0, - "7": 1878.0, - "8": 1712.0, - "9": 1784.0, - "10": 1813.0, - "11": 1724.0, - "12": 1774.0, - "13": 1798.0, - "14": 1902.0, - "15": 1579.0, - "16": 1665.0, - "17": 1839.0, - "18": 1751.0, - "19": 1684.0, - "20": 1696.0, - "21": 1813.0, - "22": 1721.0, - "23": 1705.0, - "24": 1795.0, - "25": 1699.0, - "26": 1767.0, - "27": 1747.0, - "28": 1871.0, - "29": 1847.0, - "30": 1813.0, - "31": 2027.0, - "32": 2003.0, - "33": 2032.0, - "34": 2058.0, - "35": 2136.0, - "36": 2084.0, - "37": 2158.0, - "38": 2049.0, - "39": 2259.0, - "40": 2277.0, - "41": 2445.0, - "42": 2144.0, - "43": 2428.0, - "44": 2234.0, - "45": 2590.0, - "46": 2394.0, - "47": 2567.0, - "48": 2671.0, - "49": 2833.0, - "50": 2640.0 + "1": 1711.0, + "2": 1729.0, + "3": 1743.0, + "4": 1717.0, + "5": 1753.0, + "6": 1571.0, + "7": 1945.0, + "8": 1664.0, + "9": 1763.0, + "10": 1756.0, + "11": 1576.0, + "12": 1740.0, + "13": 1769.0, + "14": 1912.0, + "15": 1642.0, + "16": 1696.0, + "17": 1758.0, + "18": 1759.0, + "19": 1606.0, + "20": 1712.0, + "21": 1851.0, + "22": 1755.0, + "23": 1724.0, + "24": 1738.0, + "25": 1784.0, + "26": 1867.0, + "27": 1786.0, + "28": 1854.0, + "29": 1875.0, + "30": 1824.0, + "31": 1920.0, + "32": 1900.0, + "33": 1953.0, + "34": 2163.0, + "35": 2095.0, + "36": 2073.0, + "37": 2097.0, + "38": 2123.0, + "39": 2225.0, + "40": 2302.0, + "41": 2269.0, + "42": 2037.0, + "43": 2399.0, + "44": 2300.0, + "45": 2662.0, + "46": 2412.0, + "47": 2565.0, + "48": 2779.0, + "49": 3046.0, + "50": 2667.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.00957, - "3": 0.1544, - "4": 0.99524, - "5": 0.2564, - "6": 0.75781, - "7": 1.04132, - "8": 1.3185, - "9": 0.67632, - "10": 0.44667, - "11": 0.97555, - "12": 0.37765, - "13": 0.64289, - "14": 0.57222, - "15": 0.50117, - "16": 0.66784, - "17": 0.5169, - "18": 0.83242, - "19": 0.2859, - "20": 0.59278, - "21": 0.69966, - "22": 0.54056, - "23": 0.60898, - "24": 0.43047, - "25": 0.45108, - "26": 0.85123, - "27": 0.20228, - "28": 0.56624, - "29": 0.74604, - "30": 0.16317, - "31": 0.42617, - "32": 0.85886, - "33": 0.5399, - "34": 0.65074, - "35": 0.4537, - "36": 0.50039, - "37": 0.48866, - "38": 0.48904, - "39": 0.62383, - "40": 0.6464, - "41": 0.60573, - "42": 0.51136, - "43": 0.49241, - "44": 0.6534, - "45": 0.59786, - "46": 0.43762, - "47": 0.56581, - "48": 0.73181, - "49": 0.37909, - "50": 0.40553 + "2": 4.01334, + "3": 0.13571, + "4": 1.30872, + "5": 0.44289, + "6": 0.70389, + "7": 0.70138, + "8": 0.95215, + "9": 0.42612, + "10": 0.49007, + "11": 1.09036, + "12": 0.70358, + "13": 0.65182, + "14": 0.60549, + "15": 0.5393, + "16": 0.69872, + "17": 0.34669, + "18": 1.04495, + "19": 0.4883, + "20": 0.35937, + "21": 0.73364, + "22": 0.78367, + "23": 0.62971, + "24": 0.34388, + "25": 0.5076, + "26": 1.09769, + "27": 0.56035, + "28": 0.58016, + "29": 0.83533, + "30": 0.16612, + "31": 0.32188, + "32": 1.11965, + "33": 0.76135, + "34": 0.37733, + "35": 0.82335, + "36": 0.69739, + "37": 0.54966, + "38": 0.53575, + "39": 0.96292, + "40": 0.90167, + "41": 0.59391, + "42": 0.71867, + "43": 0.54187, + "44": 0.67851, + "45": 0.65862, + "46": 0.47508, + "47": 0.57993, + "48": 1.31215, + "49": 0.46445, + "50": 0.33581 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json index 7e18481b88f..c6e57e04ad9 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93399, - "2": 10.92476, - "3": 10.92682, - "4": 10.93022, - "5": 10.93698, - "6": 10.92965, + "1": 10.93393, + "2": 10.92467, + "3": 10.92678, + "4": 10.93013, + "5": 10.93701, + "6": 10.9296, "7": 10.92253, - "8": 10.92212, - "9": 10.92457, - "10": 10.92887, - "11": 10.90984, - "12": 10.92854, - "13": 10.89636, - "14": 10.89646, - "15": 10.89151, - "16": 10.87388, + "8": 10.9221, + "9": 10.92459, + "10": 10.92892, + "11": 10.90982, + "12": 10.92853, + "13": 10.89638, + "14": 10.89651, + "15": 10.8915, + "16": 10.87384, "17": 10.8703, - "18": 10.8717, - "19": 10.85327, - "20": 10.80493, - "21": 10.7837, - "22": 10.7641, - "23": 10.77106, - "24": 10.75051, - "25": 10.74642, - "26": 10.72229, - "27": 10.68745, - "28": 10.61503, - "29": 10.59229, - "30": 10.55596, - "31": 10.56461, - "32": 10.54468, - "33": 10.51563, - "34": 10.48265, - "35": 10.48463, - "36": 10.46505, - "37": 10.42806, - "38": 10.42575, - "39": 10.39425, - "40": 10.37402, - "41": 10.35095, - "42": 10.34773, - "43": 10.31923, - "44": 10.29862, - "45": 10.30319, - "46": 10.26407, - "47": 10.25193, - "48": 10.215, - "49": 10.20174, - "50": 10.21031 + "18": 10.87176, + "19": 10.85332, + "20": 10.80507, + "21": 10.78369, + "22": 10.76407, + "23": 10.77113, + "24": 10.75059, + "25": 10.74651, + "26": 10.72238, + "27": 10.68742, + "28": 10.61512, + "29": 10.59231, + "30": 10.55603, + "31": 10.56468, + "32": 10.54471, + "33": 10.51566, + "34": 10.48267, + "35": 10.48467, + "36": 10.46512, + "37": 10.42808, + "38": 10.4258, + "39": 10.39431, + "40": 10.374, + "41": 10.35096, + "42": 10.34771, + "43": 10.3193, + "44": 10.29873, + "45": 10.30323, + "46": 10.26412, + "47": 10.25198, + "48": 10.21503, + "49": 10.2018, + "50": 10.21033 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1881.0, - "2": 1708.0, - "3": 1776.0, - "4": 1728.0, - "5": 1710.0, - "6": 1765.0, - "7": 2082.0, - "8": 1791.0, - "9": 1899.0, - "10": 1705.0, - "11": 1734.0, - "12": 1770.0, - "13": 1899.0, - "14": 1951.0, - "15": 1709.0, - "16": 1899.0, - "17": 1840.0, - "18": 1800.0, - "19": 1708.0, - "20": 1628.0, - "21": 1827.0, - "22": 1708.0, - "23": 1704.0, - "24": 1826.0, - "25": 1723.0, - "26": 1871.0, - "27": 1882.0, - "28": 1943.0, - "29": 2002.0, - "30": 1945.0, - "31": 2070.0, - "32": 1917.0, - "33": 2049.0, - "34": 2073.0, - "35": 2144.0, - "36": 2141.0, - "37": 2352.0, - "38": 2075.0, - "39": 2247.0, - "40": 2338.0, - "41": 2357.0, - "42": 2115.0, - "43": 2414.0, - "44": 2263.0, - "45": 2650.0, - "46": 2439.0, - "47": 2703.0, - "48": 2508.0, - "49": 2809.0, - "50": 2694.0 + "1": 1889.0, + "2": 1680.0, + "3": 1705.0, + "4": 1742.0, + "5": 1810.0, + "6": 1834.0, + "7": 2160.0, + "8": 1761.0, + "9": 1880.0, + "10": 1715.0, + "11": 1802.0, + "12": 1752.0, + "13": 1784.0, + "14": 1878.0, + "15": 1778.0, + "16": 1781.0, + "17": 1861.0, + "18": 1789.0, + "19": 1755.0, + "20": 1780.0, + "21": 1766.0, + "22": 1644.0, + "23": 1724.0, + "24": 1782.0, + "25": 1763.0, + "26": 1759.0, + "27": 1901.0, + "28": 2028.0, + "29": 1972.0, + "30": 1858.0, + "31": 2101.0, + "32": 1974.0, + "33": 1992.0, + "34": 2159.0, + "35": 2103.0, + "36": 2057.0, + "37": 2299.0, + "38": 2093.0, + "39": 2288.0, + "40": 2297.0, + "41": 2378.0, + "42": 2110.0, + "43": 2366.0, + "44": 2316.0, + "45": 2570.0, + "46": 2513.0, + "47": 2644.0, + "48": 2552.0, + "49": 2881.0, + "50": 2617.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.65413, - "3": 0.08085, - "4": 0.08064, - "5": 0.08045, - "6": 0.08256, - "7": 0.08015, - "8": 0.0813, - "9": 0.07903, - "10": 0.08042, - "11": 0.08177, - "12": 0.08108, - "13": 0.07969, - "14": 0.08025, - "15": 0.08016, - "16": 0.08011, - "17": 0.07942, - "18": 0.08033, - "19": 0.07982, - "20": 0.07968, - "21": 0.08004, - "22": 0.08175, - "23": 0.08418, - "24": 0.08754, - "25": 0.07846, - "26": 0.08009, - "27": 0.07834, - "28": 0.08125, - "29": 0.07899, - "30": 0.08056, - "31": 0.07954, - "32": 0.07996, - "33": 0.07917, - "34": 0.07986, - "35": 0.07983, - "36": 0.08227, - "37": 0.08683, - "38": 0.08139, - "39": 0.07946, - "40": 0.07988, - "41": 0.08031, - "42": 0.08136, - "43": 0.08108, - "44": 0.08087, - "45": 0.07824, - "46": 0.0806, - "47": 0.0811, - "48": 0.08044, - "49": 0.07912, - "50": 0.08042 + "2": 2.72642, + "3": 0.09059, + "4": 0.08575, + "5": 0.08449, + "6": 0.08618, + "7": 0.08348, + "8": 0.08347, + "9": 0.08304, + "10": 0.08209, + "11": 0.0825, + "12": 0.08521, + "13": 0.08578, + "14": 0.08315, + "15": 0.08315, + "16": 0.08341, + "17": 0.08327, + "18": 0.08342, + "19": 0.08845, + "20": 0.08296, + "21": 0.08379, + "22": 0.08347, + "23": 0.08279, + "24": 0.08332, + "25": 0.08609, + "26": 0.08363, + "27": 0.08249, + "28": 0.08517, + "29": 0.08438, + "30": 0.08202, + "31": 0.08249, + "32": 0.082, + "33": 0.08222, + "34": 0.08139, + "35": 0.08176, + "36": 0.0823, + "37": 0.08203, + "38": 0.08384, + "39": 0.08185, + "40": 0.08184, + "41": 0.08469, + "42": 0.0827, + "43": 0.08589, + "44": 0.08182, + "45": 0.08166, + "46": 0.08203, + "47": 0.0824, + "48": 0.08229, + "49": 0.08196, + "50": 0.08201 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json index 5a27b3f8d56..cf1dd10401f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.88533, - "2": 10.88323, - "3": 10.88389, - "4": 10.88685, + "1": 10.8853, + "2": 10.88321, + "3": 10.88381, + "4": 10.88683, "5": 10.8789, - "6": 10.87638, - "7": 10.8797, - "8": 10.87233, - "9": 10.88488, - "10": 10.87864, - "11": 10.87362, - "12": 10.86361, + "6": 10.87631, + "7": 10.87967, + "8": 10.87231, + "9": 10.88486, + "10": 10.87858, + "11": 10.87356, + "12": 10.86371, "13": 10.87879, - "14": 10.865, - "15": 10.83819, + "14": 10.86496, + "15": 10.83824, "16": 10.83255, - "17": 10.84053, - "18": 10.82214, - "19": 10.83598, - "20": 10.72905, - "21": 10.73896, - "22": 10.72498, - "23": 10.71146, - "24": 10.68267, - "25": 10.67564, - "26": 10.67938, - "27": 10.63466, - "28": 10.57945, - "29": 10.54432, - "30": 10.52102, - "31": 10.5136, - "32": 10.49426, - "33": 10.46836, - "34": 10.43371, - "35": 10.43863, - "36": 10.41292, - "37": 10.37869, - "38": 10.39311, - "39": 10.35118, - "40": 10.34578, - "41": 10.32306, - "42": 10.29734, - "43": 10.28519, - "44": 10.24436, - "45": 10.27149, - "46": 10.22645, - "47": 10.22022, - "48": 10.1732, - "49": 10.17067, - "50": 10.17205 + "17": 10.84051, + "18": 10.82218, + "19": 10.83596, + "20": 10.72909, + "21": 10.73906, + "22": 10.72491, + "23": 10.71153, + "24": 10.68273, + "25": 10.67559, + "26": 10.67939, + "27": 10.63467, + "28": 10.57956, + "29": 10.54433, + "30": 10.52116, + "31": 10.51362, + "32": 10.49421, + "33": 10.46838, + "34": 10.43376, + "35": 10.43862, + "36": 10.41297, + "37": 10.37868, + "38": 10.39314, + "39": 10.35125, + "40": 10.34579, + "41": 10.3231, + "42": 10.29733, + "43": 10.2853, + "44": 10.24443, + "45": 10.27153, + "46": 10.22648, + "47": 10.22033, + "48": 10.17323, + "49": 10.1707, + "50": 10.17214 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1752.0, - "2": 1759.0, - "3": 1797.0, - "4": 1699.0, - "5": 1723.0, - "6": 1652.0, - "7": 1826.0, - "8": 1721.0, - "9": 1738.0, - "10": 1755.0, - "11": 1713.0, - "12": 1633.0, - "13": 1838.0, - "14": 1805.0, - "15": 1624.0, - "16": 1772.0, - "17": 1803.0, - "18": 1885.0, - "19": 1669.0, - "20": 1740.0, - "21": 1750.0, - "22": 1711.0, - "23": 1656.0, - "24": 1801.0, - "25": 1665.0, - "26": 1869.0, - "27": 1901.0, - "28": 1874.0, - "29": 1934.0, - "30": 1835.0, - "31": 2054.0, - "32": 1967.0, - "33": 1859.0, - "34": 1998.0, - "35": 2070.0, - "36": 2010.0, - "37": 2252.0, - "38": 2135.0, - "39": 2189.0, - "40": 2277.0, - "41": 2394.0, - "42": 2163.0, - "43": 2421.0, - "44": 2333.0, - "45": 2614.0, - "46": 2537.0, - "47": 2494.0, - "48": 2705.0, - "49": 2869.0, - "50": 2689.0 + "1": 1806.0, + "2": 1666.0, + "3": 1783.0, + "4": 1693.0, + "5": 1654.0, + "6": 1608.0, + "7": 1873.0, + "8": 1698.0, + "9": 1693.0, + "10": 1821.0, + "11": 1696.0, + "12": 1683.0, + "13": 1736.0, + "14": 1800.0, + "15": 1622.0, + "16": 1836.0, + "17": 1808.0, + "18": 1795.0, + "19": 1633.0, + "20": 1618.0, + "21": 1776.0, + "22": 1658.0, + "23": 1687.0, + "24": 1786.0, + "25": 1651.0, + "26": 1907.0, + "27": 1859.0, + "28": 1873.0, + "29": 2007.0, + "30": 1769.0, + "31": 2107.0, + "32": 1936.0, + "33": 1999.0, + "34": 2014.0, + "35": 2117.0, + "36": 2011.0, + "37": 2288.0, + "38": 2159.0, + "39": 2281.0, + "40": 2350.0, + "41": 2358.0, + "42": 2100.0, + "43": 2369.0, + "44": 2298.0, + "45": 2651.0, + "46": 2400.0, + "47": 2499.0, + "48": 2720.0, + "49": 2940.0, + "50": 2656.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.79185, - "3": 0.1684, - "4": 0.30007, - "5": 0.41737, - "6": 0.61385, - "7": 0.4785, - "8": 0.56856, - "9": 0.89221, - "10": 0.31726, - "11": 0.62081, - "12": 0.44217, - "13": 0.3146, - "14": 0.49953, - "15": 0.31351, - "16": 0.5787, - "17": 0.39675, - "18": 0.49382, - "19": 0.25357, - "20": 0.67138, - "21": 0.34825, - "22": 0.39821, - "23": 0.33723, - "24": 0.36661, - "25": 0.65768, - "26": 0.56364, - "27": 0.49364, - "28": 0.32846, - "29": 0.3984, - "30": 0.37947, - "31": 0.19854, - "32": 0.38359, - "33": 0.40613, - "34": 0.44257, - "35": 0.69906, - "36": 0.3527, - "37": 0.36409, - "38": 0.47772, - "39": 0.38507, - "40": 0.34556, - "41": 0.44601, - "42": 0.38203, - "43": 0.4813, - "44": 0.31021, - "45": 0.3063, - "46": 0.3706, - "47": 0.40902, - "48": 0.40634, - "49": 0.63022, - "50": 0.52699 + "2": 2.75807, + "3": 0.14096, + "4": 0.47833, + "5": 0.3358, + "6": 0.52701, + "7": 0.58729, + "8": 0.56622, + "9": 0.40433, + "10": 0.39253, + "11": 0.72789, + "12": 0.4563, + "13": 0.37126, + "14": 0.57148, + "15": 0.34691, + "16": 0.50552, + "17": 0.17416, + "18": 0.54054, + "19": 0.47334, + "20": 0.83111, + "21": 0.38196, + "22": 0.27584, + "23": 0.57773, + "24": 0.2311, + "25": 0.65742, + "26": 0.26945, + "27": 0.38907, + "28": 0.38416, + "29": 0.29269, + "30": 0.3397, + "31": 0.28969, + "32": 0.41976, + "33": 0.27329, + "34": 0.19107, + "35": 0.62567, + "36": 0.29018, + "37": 0.37147, + "38": 0.28287, + "39": 0.36929, + "40": 0.46519, + "41": 0.41803, + "42": 0.34345, + "43": 0.5003, + "44": 0.4974, + "45": 0.17692, + "46": 0.53136, + "47": 0.41302, + "48": 0.24845, + "49": 0.5349, + "50": 0.41671 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 15d7bdfab04..b22d95f4701 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.8851, - "2": 10.88271, - "3": 10.88387, - "4": 10.88623, - "5": 10.87916, - "6": 10.87659, - "7": 10.87962, - "8": 10.87212, - "9": 10.88437, - "10": 10.87861, + "1": 10.88508, + "2": 10.88267, + "3": 10.88381, + "4": 10.88628, + "5": 10.87917, + "6": 10.87655, + "7": 10.87963, + "8": 10.87208, + "9": 10.88438, + "10": 10.87859, "11": 10.87351, - "12": 10.86322, - "13": 10.87875, - "14": 10.86597, - "15": 10.83786, - "16": 10.83261, - "17": 10.84042, - "18": 10.82308, - "19": 10.8357, - "20": 10.72914, - "21": 10.73909, - "22": 10.72511, - "23": 10.71158, - "24": 10.68285, - "25": 10.67542, - "26": 10.67984, - "27": 10.63495, - "28": 10.57987, + "12": 10.86335, + "13": 10.87873, + "14": 10.86596, + "15": 10.83787, + "16": 10.83256, + "17": 10.84044, + "18": 10.82313, + "19": 10.83568, + "20": 10.72916, + "21": 10.73922, + "22": 10.72505, + "23": 10.71164, + "24": 10.6829, + "25": 10.67538, + "26": 10.67983, + "27": 10.63488, + "28": 10.57997, "29": 10.54429, - "30": 10.52094, - "31": 10.51407, - "32": 10.49438, - "33": 10.46828, - "34": 10.43398, - "35": 10.43852, - "36": 10.4131, - "37": 10.3791, - "38": 10.39374, - "39": 10.35128, - "40": 10.34595, - "41": 10.32312, - "42": 10.29759, - "43": 10.28543, - "44": 10.24463, - "45": 10.27145, - "46": 10.22653, - "47": 10.22023, - "48": 10.17334, - "49": 10.17076, - "50": 10.17217, + "30": 10.52101, + "31": 10.51408, + "32": 10.49434, + "33": 10.46831, + "34": 10.43403, + "35": 10.4385, + "36": 10.41313, + "37": 10.37914, + "38": 10.39376, + "39": 10.35138, + "40": 10.34602, + "41": 10.32317, + "42": 10.29756, + "43": 10.28548, + "44": 10.24467, + "45": 10.27152, + "46": 10.22656, + "47": 10.2203, + "48": 10.17338, + "49": 10.1708, + "50": 10.17222, "51": 10.1777, - "52": 10.13082, - "53": 10.1398, - "54": 10.10412, - "55": 10.0701, - "56": 10.10629, - "57": 10.09822, - "58": 10.1116, - "59": 10.0548, - "60": 10.07318, - "61": 10.02711, - "62": 9.99676, - "63": 10.07084, - "64": 10.02892, - "65": 10.00122, - "66": 10.02707, - "67": 10.00255, - "68": 9.96484, + "52": 10.13089, + "53": 10.13982, + "54": 10.10413, + "55": 10.07015, + "56": 10.10634, + "57": 10.0982, + "58": 10.11163, + "59": 10.05487, + "60": 10.07321, + "61": 10.02714, + "62": 9.99681, + "63": 10.07086, + "64": 10.02893, + "65": 10.00125, + "66": 10.02713, + "67": 10.00263, + "68": 9.96482, "69": 9.98873, - "70": 9.97572, - "71": 9.9981, - "72": 9.97566, - "73": 9.96388, - "74": 9.95687, - "75": 9.92479, - "76": 9.9637, - "77": 9.95559, - "78": 9.90277, - "79": 9.90952, - "80": 9.92737, - "81": 9.94386, - "82": 9.89003, - "83": 9.84869, - "84": 9.78551, - "85": 9.77693, - "86": 9.87452, - "87": 9.90795, - "88": 9.88609, + "70": 9.97571, + "71": 9.99817, + "72": 9.97571, + "73": 9.96387, + "74": 9.9569, + "75": 9.92482, + "76": 9.96373, + "77": 9.95564, + "78": 9.90284, + "79": 9.90955, + "80": 9.92744, + "81": 9.94394, + "82": 9.89006, + "83": 9.84876, + "84": 9.78554, + "85": 9.77698, + "86": 9.87455, + "87": 9.908, + "88": 9.88614, "89": 9.81901, - "90": 9.81049, - "91": 9.82156, - "92": 9.81513, - "93": 9.7511, - "94": 9.82585, - "95": 9.82052, - "96": 9.80342, - "97": 9.74064, - "98": 9.77418, - "99": 9.81987, - "100": 9.71431 + "90": 9.81052, + "91": 9.82161, + "92": 9.81514, + "93": 9.75109, + "94": 9.8259, + "95": 9.82056, + "96": 9.80345, + "97": 9.74073, + "98": 9.77421, + "99": 9.8199, + "100": 9.71432 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1709.0, - "2": 1738.0, - "3": 1755.0, - "4": 1816.0, - "5": 1708.0, - "6": 1634.0, - "7": 1838.0, - "8": 1695.0, - "9": 1767.0, - "10": 1770.0, - "11": 1646.0, - "12": 1694.0, - "13": 1827.0, - "14": 1836.0, - "15": 1668.0, - "16": 1771.0, - "17": 1710.0, - "18": 1810.0, - "19": 1626.0, - "20": 1637.0, - "21": 1755.0, - "22": 1610.0, - "23": 1699.0, - "24": 1736.0, - "25": 1665.0, - "26": 1825.0, - "27": 1873.0, - "28": 1873.0, - "29": 1950.0, - "30": 1801.0, - "31": 2007.0, - "32": 1957.0, - "33": 1989.0, - "34": 2056.0, - "35": 1989.0, - "36": 1986.0, - "37": 2275.0, - "38": 2141.0, - "39": 2165.0, - "40": 2317.0, - "41": 2404.0, - "42": 2080.0, - "43": 2426.0, - "44": 2321.0, - "45": 2630.0, - "46": 2483.0, - "47": 2580.0, - "48": 2699.0, - "49": 2889.0, - "50": 2667.0, - "51": 2617.0, - "52": 2850.0, - "53": 2625.0, - "54": 3028.0, - "55": 2634.0, - "56": 2766.0, - "57": 2237.0, - "58": 3612.0, - "59": 2973.0, - "60": 2999.0, - "61": 2852.0, - "62": 3267.0, - "63": 3413.0, - "64": 3691.0, - "65": 2757.0, - "66": 3276.0, - "67": 3939.0, - "68": 3751.0, - "69": 3041.0, - "70": 3416.0, - "71": 3101.0, - "72": 3182.0, - "73": 3409.0, - "74": 3380.0, - "75": 3196.0, - "76": 3354.0, - "77": 3776.0, - "78": 3164.0, - "79": 3305.0, - "80": 2976.0, - "81": 3507.0, - "82": 3083.0, - "83": 3097.0, - "84": 3179.0, - "85": 2726.0, - "86": 3155.0, - "87": 2915.0, - "88": 2976.0, - "89": 2918.0, - "90": 3555.0, - "91": 3025.0, - "92": 3105.0, - "93": 3130.0, - "94": 2975.0, - "95": 3522.0, - "96": 3262.0, - "97": 3671.0, - "98": 3711.0, - "99": 3330.0, - "100": 3251.0 + "1": 1824.0, + "2": 1713.0, + "3": 1872.0, + "4": 1690.0, + "5": 1686.0, + "6": 1613.0, + "7": 1828.0, + "8": 1722.0, + "9": 1716.0, + "10": 1818.0, + "11": 1631.0, + "12": 1669.0, + "13": 1800.0, + "14": 1755.0, + "15": 1631.0, + "16": 1721.0, + "17": 1856.0, + "18": 1900.0, + "19": 1636.0, + "20": 1772.0, + "21": 1802.0, + "22": 1641.0, + "23": 1666.0, + "24": 1872.0, + "25": 1760.0, + "26": 1829.0, + "27": 1889.0, + "28": 1815.0, + "29": 1898.0, + "30": 1847.0, + "31": 1969.0, + "32": 1980.0, + "33": 1915.0, + "34": 2069.0, + "35": 1997.0, + "36": 2015.0, + "37": 2151.0, + "38": 2117.0, + "39": 2171.0, + "40": 2211.0, + "41": 2322.0, + "42": 2104.0, + "43": 2449.0, + "44": 2326.0, + "45": 2713.0, + "46": 2396.0, + "47": 2602.0, + "48": 2663.0, + "49": 2932.0, + "50": 2675.0, + "51": 2588.0, + "52": 2858.0, + "53": 2618.0, + "54": 3000.0, + "55": 2710.0, + "56": 2785.0, + "57": 2223.0, + "58": 3664.0, + "59": 2943.0, + "60": 2942.0, + "61": 2776.0, + "62": 3178.0, + "63": 3484.0, + "64": 3783.0, + "65": 2726.0, + "66": 3229.0, + "67": 3944.0, + "68": 3708.0, + "69": 3014.0, + "70": 3465.0, + "71": 3161.0, + "72": 3092.0, + "73": 3469.0, + "74": 3377.0, + "75": 3166.0, + "76": 3304.0, + "77": 3749.0, + "78": 3340.0, + "79": 3161.0, + "80": 3000.0, + "81": 3578.0, + "82": 3152.0, + "83": 3167.0, + "84": 3054.0, + "85": 2736.0, + "86": 3135.0, + "87": 2983.0, + "88": 3049.0, + "89": 2953.0, + "90": 3523.0, + "91": 2958.0, + "92": 3151.0, + "93": 3191.0, + "94": 3151.0, + "95": 3498.0, + "96": 3354.0, + "97": 3654.0, + "98": 3692.0, + "99": 3368.0, + "100": 3213.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.76083, - "3": 0.1392, - "4": 0.72471, - "5": 0.37212, - "6": 0.65496, - "7": 0.81145, - "8": 1.18276, - "9": 0.74068, - "10": 0.62362, - "11": 0.90016, - "12": 0.3931, - "13": 0.66456, - "14": 0.54956, - "15": 0.5344, - "16": 0.56924, - "17": 0.48379, - "18": 0.82846, - "19": 0.52198, - "20": 0.30559, - "21": 0.68469, - "22": 0.6055, - "23": 0.6241, - "24": 0.74674, - "25": 0.14371, - "26": 0.74475, - "27": 0.24564, - "28": 0.64924, - "29": 0.62848, - "30": 0.15121, - "31": 0.51775, - "32": 1.05545, - "33": 0.36409, - "34": 0.82885, - "35": 0.18612, - "36": 0.70268, - "37": 0.53875, - "38": 0.41068, - "39": 0.37853, - "40": 0.56174, - "41": 0.71995, - "42": 0.46478, - "43": 0.48267, - "44": 1.00388, - "45": 0.19292, - "46": 0.6001, - "47": 0.51524, - "48": 0.72325, - "49": 0.4162, - "50": 0.39522, - "51": 0.17241, - "52": 0.61332, - "53": 0.52274, - "54": 0.74178, - "55": 0.54106, - "56": 0.80779, - "57": 0.43858, - "58": 0.51628, - "59": 0.72496, - "60": 0.66828, - "61": 0.27282, - "62": 0.75232, - "63": 0.56444, - "64": 0.65677, - "65": 0.48561, - "66": 0.80335, - "67": 0.65147, - "68": 0.15498, - "69": 0.42658, - "70": 1.05728, - "71": 0.2845, - "72": 0.64147, - "73": 0.57041, - "74": 0.44787, - "75": 0.97937, - "76": 0.45986, - "77": 0.41157, - "78": 1.43496, - "79": 0.3979, - "80": 0.55705, - "81": 0.65481, - "82": 0.3819, - "83": 0.66336, - "84": 0.74797, - "85": 0.14404, - "86": 0.46607, - "87": 1.04081, - "88": 0.53513, - "89": 0.79143, - "90": 0.46305, - "91": 0.49708, - "92": 0.70755, - "93": 0.25051, - "94": 0.58557, - "95": 1.0549, - "96": 0.4588, - "97": 0.58513, - "98": 0.64325, - "99": 1.33531, - "100": 0.30734 + "2": 4.04879, + "3": 0.1206, + "4": 0.83583, + "5": 0.3982, + "6": 0.703, + "7": 0.93185, + "8": 1.09606, + "9": 0.70874, + "10": 0.35186, + "11": 1.0367, + "12": 0.38565, + "13": 0.68739, + "14": 0.63959, + "15": 0.58274, + "16": 0.7569, + "17": 0.64911, + "18": 0.84129, + "19": 0.4851, + "20": 0.38822, + "21": 0.85612, + "22": 0.68806, + "23": 0.66893, + "24": 0.68341, + "25": 0.1606, + "26": 0.97351, + "27": 0.30022, + "28": 0.61744, + "29": 0.81653, + "30": 0.12422, + "31": 0.36082, + "32": 0.99296, + "33": 0.52091, + "34": 0.39913, + "35": 0.7243, + "36": 0.54103, + "37": 0.71705, + "38": 0.47887, + "39": 0.65177, + "40": 0.79721, + "41": 0.81076, + "42": 0.38399, + "43": 0.66645, + "44": 0.66024, + "45": 0.6778, + "46": 0.43432, + "47": 0.59492, + "48": 0.8034, + "49": 0.48471, + "50": 0.36363, + "51": 0.17609, + "52": 0.80964, + "53": 0.53817, + "54": 0.57625, + "55": 0.73199, + "56": 0.68396, + "57": 0.41288, + "58": 0.61013, + "59": 0.57041, + "60": 0.59687, + "61": 0.30585, + "62": 1.01408, + "63": 0.18035, + "64": 0.97531, + "65": 0.37541, + "66": 0.74098, + "67": 0.61337, + "68": 0.12641, + "69": 0.65062, + "70": 1.00918, + "71": 0.2935, + "72": 0.69945, + "73": 0.59096, + "74": 0.45528, + "75": 0.97305, + "76": 0.4391, + "77": 0.45408, + "78": 1.44297, + "79": 0.32136, + "80": 0.6121, + "81": 0.67269, + "82": 0.34379, + "83": 0.64038, + "84": 0.76052, + "85": 0.20412, + "86": 0.44895, + "87": 1.02436, + "88": 0.47849, + "89": 0.76797, + "90": 0.50489, + "91": 0.42757, + "92": 0.68449, + "93": 0.43802, + "94": 0.45655, + "95": 0.95781, + "96": 0.56689, + "97": 0.49148, + "98": 0.61546, + "99": 1.34482, + "100": 0.33349 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json index 0e05742094a..278a254ffa5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93345, - "2": 10.92442, - "3": 10.92705, - "4": 10.93095, - "5": 10.93662, - "6": 10.92967, - "7": 10.92359, - "8": 10.92254, - "9": 10.92459, - "10": 10.92964, - "11": 10.90935, - "12": 10.9283, - "13": 10.89685, - "14": 10.89709, - "15": 10.89153, - "16": 10.87452, - "17": 10.87088, - "18": 10.87178, + "1": 10.93341, + "2": 10.92431, + "3": 10.92704, + "4": 10.93087, + "5": 10.93656, + "6": 10.92961, + "7": 10.9236, + "8": 10.92262, + "9": 10.92457, + "10": 10.92966, + "11": 10.90934, + "12": 10.92833, + "13": 10.89681, + "14": 10.89713, + "15": 10.89155, + "16": 10.87453, + "17": 10.8709, + "18": 10.87184, "19": 10.85369, - "20": 10.8046, - "21": 10.78432, - "22": 10.76481, - "23": 10.77092, - "24": 10.75018, - "25": 10.74709, + "20": 10.80471, + "21": 10.78433, + "22": 10.76479, + "23": 10.77096, + "24": 10.75026, + "25": 10.7471, "26": 10.72265, - "27": 10.68797, - "28": 10.61498, - "29": 10.59319, - "30": 10.55627, - "31": 10.56492, - "32": 10.54516, - "33": 10.51564, - "34": 10.48222, - "35": 10.48502, - "36": 10.46535, - "37": 10.42861, - "38": 10.42594, - "39": 10.39452, - "40": 10.3745, - "41": 10.3512, - "42": 10.34808, - "43": 10.31967, - "44": 10.29914, - "45": 10.30312, - "46": 10.26444, - "47": 10.25216, - "48": 10.21538, - "49": 10.202, - "50": 10.21083, - "51": 10.19946, - "52": 10.15802, - "53": 10.1637, - "54": 10.12833, - "55": 10.10244, - "56": 10.12752, - "57": 10.11583, - "58": 10.11961, - "59": 10.06695, - "60": 10.09292, - "61": 10.04269, - "62": 10.01398, - "63": 10.08126, - "64": 10.03224, - "65": 10.00032, - "66": 10.04316, - "67": 10.01552, - "68": 9.98467, - "69": 9.99675, + "27": 10.68792, + "28": 10.61507, + "29": 10.59321, + "30": 10.55636, + "31": 10.56495, + "32": 10.54518, + "33": 10.51569, + "34": 10.48225, + "35": 10.48504, + "36": 10.46545, + "37": 10.42864, + "38": 10.42608, + "39": 10.3946, + "40": 10.37454, + "41": 10.35122, + "42": 10.3481, + "43": 10.31972, + "44": 10.29918, + "45": 10.30317, + "46": 10.26454, + "47": 10.25224, + "48": 10.21545, + "49": 10.2021, + "50": 10.21085, + "51": 10.19943, + "52": 10.15812, + "53": 10.16371, + "54": 10.12837, + "55": 10.10246, + "56": 10.12757, + "57": 10.11585, + "58": 10.11966, + "59": 10.06692, + "60": 10.09295, + "61": 10.04272, + "62": 10.01401, + "63": 10.08132, + "64": 10.03233, + "65": 10.00041, + "66": 10.04317, + "67": 10.01559, + "68": 9.98468, + "69": 9.99673, "70": 9.97884, - "71": 10.00317, - "72": 9.99337, - "73": 9.97487, - "74": 9.96899, - "75": 9.93347, - "76": 9.96, - "77": 9.95963, - "78": 9.91408, - "79": 9.91446, - "80": 9.9278, - "81": 9.95364, - "82": 9.89077, - "83": 9.85458, - "84": 9.79534, - "85": 9.78077, - "86": 9.88652, - "87": 9.90728, - "88": 9.88177, - "89": 9.82134, - "90": 9.81596, - "91": 9.82611, - "92": 9.81644, - "93": 9.74789, - "94": 9.82692, - "95": 9.8124, - "96": 9.80068, - "97": 9.74582, - "98": 9.76759, - "99": 9.82081, - "100": 9.70464 + "71": 10.00321, + "72": 9.99341, + "73": 9.97492, + "74": 9.969, + "75": 9.93351, + "76": 9.96005, + "77": 9.95969, + "78": 9.91414, + "79": 9.91454, + "80": 9.92785, + "81": 9.95369, + "82": 9.89085, + "83": 9.85463, + "84": 9.79536, + "85": 9.78085, + "86": 9.88661, + "87": 9.9073, + "88": 9.8818, + "89": 9.82136, + "90": 9.816, + "91": 9.82615, + "92": 9.81647, + "93": 9.74796, + "94": 9.82703, + "95": 9.81241, + "96": 9.8007, + "97": 9.74586, + "98": 9.76766, + "99": 9.8209, + "100": 9.70466 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1860.0, - "2": 1723.0, - "3": 1744.0, - "4": 1706.0, - "5": 1728.0, - "6": 1819.0, - "7": 2115.0, - "8": 1842.0, - "9": 1780.0, - "10": 1833.0, - "11": 1729.0, - "12": 1700.0, - "13": 1796.0, - "14": 1892.0, - "15": 1685.0, - "16": 1830.0, - "17": 1776.0, - "18": 1808.0, - "19": 1773.0, - "20": 1677.0, - "21": 2010.0, - "22": 1730.0, - "23": 1757.0, - "24": 1871.0, - "25": 1729.0, - "26": 1948.0, - "27": 1828.0, - "28": 1936.0, - "29": 2089.0, - "30": 1939.0, - "31": 2035.0, - "32": 2110.0, - "33": 2069.0, - "34": 2052.0, - "35": 2140.0, - "36": 2138.0, - "37": 2316.0, - "38": 2220.0, - "39": 2331.0, - "40": 2410.0, - "41": 2521.0, - "42": 2051.0, - "43": 2356.0, - "44": 2270.0, - "45": 2659.0, - "46": 2383.0, - "47": 2670.0, - "48": 2604.0, - "49": 2861.0, - "50": 2560.0, - "51": 2702.0, - "52": 2710.0, - "53": 2608.0, - "54": 2882.0, - "55": 2566.0, - "56": 2846.0, - "57": 2382.0, - "58": 3522.0, - "59": 2982.0, - "60": 2976.0, - "61": 2778.0, - "62": 3235.0, - "63": 3319.0, - "64": 3725.0, - "65": 2682.0, - "66": 3128.0, - "67": 3676.0, - "68": 3436.0, - "69": 3089.0, - "70": 3343.0, - "71": 3411.0, - "72": 3064.0, - "73": 3578.0, - "74": 3498.0, - "75": 3220.0, - "76": 3470.0, - "77": 3781.0, - "78": 3435.0, - "79": 3325.0, - "80": 3168.0, - "81": 3600.0, - "82": 3084.0, - "83": 3292.0, - "84": 3021.0, - "85": 2762.0, - "86": 3196.0, - "87": 2781.0, - "88": 3162.0, - "89": 3267.0, - "90": 3876.0, - "91": 2952.0, - "92": 3211.0, - "93": 3445.0, - "94": 3375.0, - "95": 3328.0, - "96": 3457.0, - "97": 3632.0, - "98": 3494.0, - "99": 3069.0, - "100": 3360.0 + "1": 1897.0, + "2": 1693.0, + "3": 1785.0, + "4": 1691.0, + "5": 1765.0, + "6": 1832.0, + "7": 2153.0, + "8": 1834.0, + "9": 1751.0, + "10": 1791.0, + "11": 1778.0, + "12": 1680.0, + "13": 1806.0, + "14": 1915.0, + "15": 1720.0, + "16": 1841.0, + "17": 1811.0, + "18": 1776.0, + "19": 1723.0, + "20": 1726.0, + "21": 1832.0, + "22": 1713.0, + "23": 1723.0, + "24": 1819.0, + "25": 1703.0, + "26": 1877.0, + "27": 1806.0, + "28": 1938.0, + "29": 1995.0, + "30": 1919.0, + "31": 2005.0, + "32": 1965.0, + "33": 1994.0, + "34": 2076.0, + "35": 2122.0, + "36": 2117.0, + "37": 2351.0, + "38": 2115.0, + "39": 2246.0, + "40": 2414.0, + "41": 2365.0, + "42": 2069.0, + "43": 2425.0, + "44": 2293.0, + "45": 2609.0, + "46": 2531.0, + "47": 2522.0, + "48": 2574.0, + "49": 2867.0, + "50": 2583.0, + "51": 2634.0, + "52": 2795.0, + "53": 2595.0, + "54": 2824.0, + "55": 2624.0, + "56": 2765.0, + "57": 2296.0, + "58": 3650.0, + "59": 2935.0, + "60": 2986.0, + "61": 2742.0, + "62": 3163.0, + "63": 3254.0, + "64": 3652.0, + "65": 2725.0, + "66": 3111.0, + "67": 3787.0, + "68": 3519.0, + "69": 3026.0, + "70": 3419.0, + "71": 3449.0, + "72": 3011.0, + "73": 3577.0, + "74": 3419.0, + "75": 3242.0, + "76": 3559.0, + "77": 3865.0, + "78": 3384.0, + "79": 3317.0, + "80": 3183.0, + "81": 3667.0, + "82": 3210.0, + "83": 3425.0, + "84": 3026.0, + "85": 2825.0, + "86": 3139.0, + "87": 2908.0, + "88": 3234.0, + "89": 3210.0, + "90": 3866.0, + "91": 3020.0, + "92": 3238.0, + "93": 3394.0, + "94": 3383.0, + "95": 3258.0, + "96": 3419.0, + "97": 3694.0, + "98": 3517.0, + "99": 3076.0, + "100": 3275.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.44354, - "3": 0.1046, - "4": 0.10023, - "5": 0.10043, - "6": 3.02749, - "7": 0.5132, - "8": 1.15763, - "9": 0.90807, - "10": 1.94768, - "11": 0.10495, - "12": 0.10128, - "13": 0.09981, - "14": 0.71606, - "15": 1.59466, - "16": 0.10639, - "17": 1.80523, - "18": 1.2658, - "19": 0.30467, - "20": 0.47372, - "21": 0.10294, - "22": 2.03265, - "23": 0.10325, - "24": 0.09979, - "25": 0.53526, - "26": 0.10429, - "27": 1.60451, - "28": 0.11321, - "29": 0.68181, - "30": 0.10314, - "31": 1.54877, - "32": 0.27113, - "33": 1.29629, - "34": 0.87885, - "35": 0.10578, - "36": 0.31464, - "37": 0.98385, - "38": 0.1039, - "39": 0.94911, - "40": 0.10301, - "41": 1.47041, - "42": 0.31144, - "43": 0.6392, - "44": 0.11646, - "45": 1.05167, - "46": 0.11092, - "47": 1.51757, - "48": 0.19002, - "49": 1.24015, - "50": 0.10714, - "51": 0.1195, - "52": 0.1297, - "53": 0.10688, - "54": 0.10433, - "55": 1.40413, - "56": 0.78418, - "57": 0.53753, - "58": 0.10316, - "59": 1.71643, - "60": 0.10158, - "61": 0.53293, - "62": 0.10557, - "63": 0.70923, - "64": 0.14978, - "65": 0.52514, - "66": 1.56714, - "67": 0.10283, - "68": 1.07961, - "69": 0.34282, - "70": 0.54304, - "71": 0.10893, - "72": 0.25115, - "73": 0.48304, - "74": 0.22828, - "75": 0.65763, - "76": 0.21947, - "77": 1.34795, - "78": 0.10849, - "79": 0.68508, - "80": 0.51653, - "81": 0.51667, - "82": 0.25543, - "83": 0.37617, - "84": 0.60505, - "85": 0.44639, - "86": 0.90911, - "87": 0.53142, - "88": 0.34676, - "89": 0.76491, - "90": 0.97605, - "91": 0.10664, - "92": 1.10868, - "93": 0.10949, - "94": 0.10162, - "95": 0.10246, - "96": 0.10562, - "97": 0.68338, - "98": 1.0847, - "99": 0.62424, - "100": 0.27861 + "2": 3.07097, + "3": 0.11248, + "4": 0.11178, + "5": 0.10994, + "6": 0.11179, + "7": 0.1098, + "8": 0.11165, + "9": 0.11009, + "10": 0.11247, + "11": 0.10983, + "12": 0.11045, + "13": 0.10953, + "14": 0.1113, + "15": 0.11098, + "16": 0.11105, + "17": 0.10971, + "18": 0.11017, + "19": 0.11056, + "20": 0.11157, + "21": 0.1107, + "22": 0.11101, + "23": 0.10948, + "24": 0.1112, + "25": 0.10867, + "26": 0.10936, + "27": 0.109, + "28": 0.10882, + "29": 0.11047, + "30": 0.10962, + "31": 0.11155, + "32": 0.11049, + "33": 0.10961, + "34": 0.10991, + "35": 0.10924, + "36": 0.1096, + "37": 0.11453, + "38": 0.11177, + "39": 0.11099, + "40": 0.10937, + "41": 0.11016, + "42": 0.10988, + "43": 0.10981, + "44": 0.10948, + "45": 0.10977, + "46": 0.10969, + "47": 0.11121, + "48": 0.10971, + "49": 0.11035, + "50": 0.10978, + "51": 0.13882, + "52": 0.13672, + "53": 0.11701, + "54": 0.11608, + "55": 0.11205, + "56": 0.11242, + "57": 0.10982, + "58": 0.11108, + "59": 0.10982, + "60": 0.111, + "61": 0.1097, + "62": 0.11184, + "63": 0.10838, + "64": 0.10954, + "65": 0.10939, + "66": 0.1101, + "67": 0.11138, + "68": 0.11124, + "69": 0.1076, + "70": 0.11134, + "71": 0.10922, + "72": 0.10898, + "73": 0.10808, + "74": 0.11004, + "75": 0.1099, + "76": 0.10853, + "77": 0.10782, + "78": 0.1086, + "79": 0.10822, + "80": 0.11118, + "81": 0.10882, + "82": 0.10898, + "83": 0.10899, + "84": 0.10857, + "85": 0.10882, + "86": 0.11144, + "87": 0.10791, + "88": 0.10871, + "89": 0.1073, + "90": 0.10758, + "91": 0.10945, + "92": 0.11193, + "93": 0.10852, + "94": 0.10835, + "95": 0.10839, + "96": 0.10753, + "97": 0.10768, + "98": 0.10854, + "99": 0.10393, + "100": 0.1079 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json index 6b787dc554f..8a405cef585 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.8851, - "2": 10.88271, - "3": 10.88387, - "4": 10.88623, - "5": 10.87916, - "6": 10.87661, + "1": 10.88508, + "2": 10.88267, + "3": 10.88381, + "4": 10.88628, + "5": 10.87917, + "6": 10.87655, "7": 10.87963, - "8": 10.87211, - "9": 10.88437, + "8": 10.87207, + "9": 10.88443, "10": 10.87858, - "11": 10.87354, - "12": 10.86326, - "13": 10.87879, - "14": 10.86595, - "15": 10.83782, - "16": 10.83261, - "17": 10.84044, - "18": 10.82306, - "19": 10.83569, + "11": 10.87347, + "12": 10.86336, + "13": 10.87875, + "14": 10.86592, + "15": 10.83783, + "16": 10.83257, + "17": 10.84046, + "18": 10.82313, + "19": 10.83571, "20": 10.72913, - "21": 10.7391, - "22": 10.72513, - "23": 10.71154, - "24": 10.68292, + "21": 10.73919, + "22": 10.72509, + "23": 10.71164, + "24": 10.68288, "25": 10.67538, - "26": 10.67981, - "27": 10.63493, - "28": 10.57986, + "26": 10.67985, + "27": 10.63489, + "28": 10.57994, "29": 10.54428, - "30": 10.5209, + "30": 10.52099, "31": 10.51407, - "32": 10.49437, - "33": 10.46826, - "34": 10.43401, - "35": 10.43851, - "36": 10.41311, - "37": 10.3791, - "38": 10.39376, - "39": 10.3513, - "40": 10.34595, - "41": 10.32313, - "42": 10.29756, - "43": 10.28542, - "44": 10.24461, - "45": 10.27146, - "46": 10.22651, - "47": 10.22023, - "48": 10.1733, - "49": 10.17078, - "50": 10.17217, - "51": 10.17766, - "52": 10.13083, - "53": 10.13975, - "54": 10.10412, - "55": 10.0701, - "56": 10.10628, - "57": 10.09819, - "58": 10.11158, - "59": 10.0548, - "60": 10.07314, - "61": 10.0271, - "62": 9.99675, + "32": 10.49436, + "33": 10.46832, + "34": 10.43403, + "35": 10.43854, + "36": 10.41312, + "37": 10.37911, + "38": 10.39378, + "39": 10.35137, + "40": 10.34605, + "41": 10.32321, + "42": 10.29759, + "43": 10.28549, + "44": 10.24468, + "45": 10.27149, + "46": 10.22659, + "47": 10.22032, + "48": 10.17336, + "49": 10.1708, + "50": 10.17222, + "51": 10.17772, + "52": 10.13088, + "53": 10.13981, + "54": 10.10415, + "55": 10.07012, + "56": 10.10633, + "57": 10.09818, + "58": 10.11161, + "59": 10.05485, + "60": 10.07321, + "61": 10.02715, + "62": 9.9968, "63": 10.07086, - "64": 10.02893, - "65": 10.00122, - "66": 10.02706, - "67": 10.00259, - "68": 9.96482, - "69": 9.9887, - "70": 9.97569, - "71": 9.99808, - "72": 9.97568, - "73": 9.96384, - "74": 9.95689, - "75": 9.92479, - "76": 9.9637, - "77": 9.95559, - "78": 9.90278, - "79": 9.90952, - "80": 9.92737, - "81": 9.94386, - "82": 9.89002, - "83": 9.84872, - "84": 9.78548, - "85": 9.77694, - "86": 9.87451, - "87": 9.90794, - "88": 9.88606, - "89": 9.81898, - "90": 9.81048, - "91": 9.82157, - "92": 9.8151, + "64": 10.02891, + "65": 10.00125, + "66": 10.02715, + "67": 10.00263, + "68": 9.96481, + "69": 9.98878, + "70": 9.97572, + "71": 9.99816, + "72": 9.97569, + "73": 9.96388, + "74": 9.95691, + "75": 9.92482, + "76": 9.96375, + "77": 9.95564, + "78": 9.90287, + "79": 9.90957, + "80": 9.92742, + "81": 9.9439, + "82": 9.89003, + "83": 9.84876, + "84": 9.78555, + "85": 9.77697, + "86": 9.87454, + "87": 9.908, + "88": 9.88615, + "89": 9.819, + "90": 9.81051, + "91": 9.82164, + "92": 9.81516, "93": 9.7511, - "94": 9.82585, - "95": 9.8205, - "96": 9.80342, - "97": 9.74065, - "98": 9.77415, - "99": 9.81989, - "100": 9.71431 + "94": 9.82591, + "95": 9.82053, + "96": 9.80346, + "97": 9.74068, + "98": 9.77419, + "99": 9.8199, + "100": 9.71434 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1709.0, - "2": 1738.0, - "3": 1755.0, - "4": 1816.0, - "5": 1698.0, - "6": 1631.0, - "7": 1869.0, - "8": 1739.0, - "9": 1691.0, - "10": 1826.0, - "11": 1722.0, - "12": 1730.0, - "13": 1818.0, - "14": 1863.0, - "15": 1615.0, - "16": 1836.0, - "17": 1747.0, - "18": 1833.0, - "19": 1608.0, - "20": 1657.0, - "21": 1859.0, - "22": 1730.0, - "23": 1635.0, - "24": 1832.0, - "25": 1729.0, - "26": 1798.0, - "27": 1878.0, - "28": 1847.0, - "29": 1947.0, - "30": 1779.0, - "31": 2074.0, - "32": 1982.0, - "33": 2070.0, - "34": 1922.0, - "35": 2024.0, - "36": 2020.0, - "37": 2265.0, - "38": 2093.0, - "39": 2265.0, - "40": 2284.0, - "41": 2386.0, - "42": 2091.0, - "43": 2397.0, - "44": 2272.0, - "45": 2554.0, - "46": 2392.0, - "47": 2509.0, - "48": 2705.0, - "49": 2865.0, - "50": 2674.0, - "51": 2545.0, - "52": 2776.0, - "53": 2583.0, - "54": 3041.0, - "55": 2759.0, - "56": 2815.0, - "57": 2403.0, - "58": 3632.0, - "59": 2959.0, - "60": 3070.0, - "61": 2800.0, - "62": 3300.0, - "63": 3522.0, - "64": 3671.0, - "65": 2744.0, - "66": 3233.0, - "67": 3924.0, - "68": 3752.0, - "69": 3066.0, - "70": 3516.0, - "71": 3069.0, - "72": 3063.0, - "73": 3411.0, - "74": 3374.0, - "75": 3206.0, - "76": 3251.0, - "77": 3808.0, - "78": 3312.0, - "79": 3270.0, - "80": 3004.0, - "81": 3497.0, - "82": 3130.0, - "83": 3054.0, - "84": 3133.0, - "85": 2843.0, - "86": 3235.0, - "87": 2937.0, - "88": 2995.0, - "89": 2953.0, - "90": 3492.0, - "91": 2867.0, - "92": 3067.0, - "93": 3202.0, - "94": 3100.0, - "95": 3381.0, - "96": 3427.0, - "97": 3692.0, - "98": 3619.0, - "99": 3378.0, - "100": 3170.0 + "1": 1824.0, + "2": 1713.0, + "3": 1872.0, + "4": 1690.0, + "5": 1686.0, + "6": 1613.0, + "7": 1828.0, + "8": 1740.0, + "9": 1761.0, + "10": 1718.0, + "11": 1674.0, + "12": 1721.0, + "13": 1855.0, + "14": 1832.0, + "15": 1623.0, + "16": 1675.0, + "17": 1829.0, + "18": 1877.0, + "19": 1661.0, + "20": 1668.0, + "21": 1791.0, + "22": 1691.0, + "23": 1676.0, + "24": 1798.0, + "25": 1659.0, + "26": 1858.0, + "27": 1851.0, + "28": 1863.0, + "29": 1878.0, + "30": 1877.0, + "31": 2045.0, + "32": 1918.0, + "33": 1977.0, + "34": 2058.0, + "35": 1992.0, + "36": 1963.0, + "37": 2267.0, + "38": 2114.0, + "39": 2218.0, + "40": 2258.0, + "41": 2353.0, + "42": 2072.0, + "43": 2406.0, + "44": 2273.0, + "45": 2683.0, + "46": 2461.0, + "47": 2463.0, + "48": 2636.0, + "49": 2904.0, + "50": 2602.0, + "51": 2624.0, + "52": 2797.0, + "53": 2568.0, + "54": 2935.0, + "55": 2728.0, + "56": 2948.0, + "57": 2298.0, + "58": 3679.0, + "59": 2956.0, + "60": 2922.0, + "61": 2807.0, + "62": 3241.0, + "63": 3504.0, + "64": 3682.0, + "65": 2727.0, + "66": 3156.0, + "67": 3950.0, + "68": 3649.0, + "69": 2946.0, + "70": 3482.0, + "71": 3226.0, + "72": 2982.0, + "73": 3523.0, + "74": 3385.0, + "75": 3168.0, + "76": 3320.0, + "77": 3701.0, + "78": 3334.0, + "79": 3194.0, + "80": 2976.0, + "81": 3505.0, + "82": 3045.0, + "83": 3097.0, + "84": 3067.0, + "85": 2814.0, + "86": 3264.0, + "87": 2959.0, + "88": 3047.0, + "89": 2921.0, + "90": 3453.0, + "91": 2971.0, + "92": 3022.0, + "93": 3243.0, + "94": 3117.0, + "95": 3551.0, + "96": 3367.0, + "97": 3815.0, + "98": 3809.0, + "99": 3355.0, + "100": 3176.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.28347, - "3": 0.20931, - "4": 0.82816, - "5": 0.42915, - "6": 0.68146, - "7": 0.58118, - "8": 0.99774, - "9": 0.61025, - "10": 0.35195, - "11": 0.97717, - "12": 0.3039, - "13": 0.64594, - "14": 0.64661, - "15": 0.38022, - "16": 0.873, - "17": 0.44623, - "18": 0.83025, - "19": 0.23953, - "20": 0.64854, - "21": 0.7005, - "22": 0.51888, - "23": 0.60687, - "24": 0.50759, - "25": 0.31193, - "26": 1.08874, - "27": 0.2125, - "28": 0.58008, - "29": 0.76487, - "30": 0.20537, - "31": 0.20034, - "32": 1.09629, - "33": 0.46348, - "34": 0.74156, - "35": 0.2114, - "36": 0.58307, - "37": 0.56521, - "38": 0.46852, - "39": 0.59962, - "40": 0.73748, - "41": 0.5621, - "42": 0.51396, - "43": 0.55903, - "44": 0.61807, - "45": 0.61512, - "46": 0.44588, - "47": 0.55796, - "48": 0.72719, - "49": 0.41084, - "50": 0.32797, - "51": 0.24383, - "52": 0.31474, - "53": 0.60589, - "54": 0.7403, - "55": 0.56813, - "56": 0.65665, - "57": 0.3942, - "58": 0.47181, - "59": 0.58245, - "60": 0.59921, - "61": 0.24895, - "62": 0.94832, - "63": 0.2419, - "64": 0.87643, - "65": 0.38117, - "66": 0.65093, - "67": 0.55319, - "68": 0.20829, - "69": 0.31956, - "70": 1.05681, - "71": 0.20846, - "72": 0.6042, - "73": 0.53133, - "74": 0.36375, - "75": 0.89679, - "76": 0.40789, - "77": 0.38711, - "78": 1.26522, - "79": 0.31147, - "80": 0.52379, - "81": 0.60742, - "82": 0.38747, - "83": 0.61588, - "84": 0.69754, - "85": 0.20763, - "86": 0.33517, - "87": 0.94717, - "88": 0.58336, - "89": 0.69931, - "90": 0.37694, - "91": 0.4405, - "92": 0.60648, - "93": 0.22284, - "94": 0.65936, - "95": 0.76711, - "96": 0.35046, - "97": 0.56031, - "98": 0.51784, - "99": 1.25475, - "100": 0.27365 + "2": 3.83224, + "3": 0.17498, + "4": 0.72445, + "5": 0.1789, + "6": 0.60257, + "7": 0.82476, + "8": 0.87772, + "9": 0.58245, + "10": 0.2957, + "11": 0.9128, + "12": 0.30011, + "13": 0.63263, + "14": 0.55001, + "15": 0.4903, + "16": 0.72692, + "17": 0.52372, + "18": 0.75816, + "19": 0.2958, + "20": 0.44797, + "21": 0.67332, + "22": 0.58992, + "23": 0.56283, + "24": 0.62735, + "25": 0.17622, + "26": 0.90391, + "27": 0.26565, + "28": 0.61527, + "29": 0.68974, + "30": 0.18041, + "31": 0.37606, + "32": 0.88114, + "33": 0.51939, + "34": 0.36074, + "35": 0.53264, + "36": 0.60397, + "37": 0.50898, + "38": 0.39492, + "39": 0.57294, + "40": 0.8355, + "41": 0.57325, + "42": 0.63972, + "43": 0.58927, + "44": 0.8033, + "45": 0.50804, + "46": 0.5082, + "47": 0.54212, + "48": 0.84587, + "49": 0.44545, + "50": 0.38373, + "51": 0.2081, + "52": 0.58115, + "53": 0.61572, + "54": 0.69775, + "55": 0.59335, + "56": 0.92543, + "57": 0.4221, + "58": 0.47128, + "59": 0.74606, + "60": 0.63497, + "61": 0.32298, + "62": 1.02608, + "63": 0.27742, + "64": 0.95438, + "65": 0.47702, + "66": 0.71715, + "67": 0.65823, + "68": 0.17672, + "69": 0.62696, + "70": 1.12762, + "71": 0.41597, + "72": 0.68256, + "73": 0.54478, + "74": 0.43496, + "75": 1.0463, + "76": 0.42072, + "77": 0.45153, + "78": 1.72692, + "79": 0.25277, + "80": 0.58024, + "81": 0.66239, + "82": 0.51575, + "83": 0.65592, + "84": 0.81146, + "85": 0.17718, + "86": 0.39647, + "87": 1.03639, + "88": 0.66652, + "89": 0.73308, + "90": 0.41386, + "91": 0.44328, + "92": 0.74923, + "93": 0.37533, + "94": 0.51843, + "95": 1.10891, + "96": 0.44707, + "97": 0.59325, + "98": 0.60289, + "99": 1.28943, + "100": 0.20128 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json index 2cca7545f14..b0484071d78 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96096, + "1": 10.96095, "2": 10.94378, - "3": 10.93765, - "4": 10.95048, - "5": 10.9484, - "6": 10.94213, - "7": 10.93466, - "8": 10.9418, - "9": 10.94159, - "10": 10.9279, - "11": 10.93407, - "12": 10.94464, - "13": 10.92188, - "14": 10.92239, - "15": 10.89966, - "16": 10.89036, - "17": 10.88908, + "3": 10.9376, + "4": 10.95053, + "5": 10.94832, + "6": 10.94216, + "7": 10.93458, + "8": 10.94185, + "9": 10.94157, + "10": 10.92794, + "11": 10.93401, + "12": 10.94467, + "13": 10.92192, + "14": 10.92242, + "15": 10.89964, + "16": 10.8904, + "17": 10.88905, "18": 10.8818, - "19": 10.87644, - "20": 10.80072, - "21": 10.78156, - "22": 10.76187, - "23": 10.76245, - "24": 10.73665, - "25": 10.73432, - "26": 10.72099, - "27": 10.68134, - "28": 10.59567, - "29": 10.57425, - "30": 10.55166, - "31": 10.55268, - "32": 10.52959, + "19": 10.87634, + "20": 10.80079, + "21": 10.78165, + "22": 10.76191, + "23": 10.76255, + "24": 10.73668, + "25": 10.73435, + "26": 10.72101, + "27": 10.6814, + "28": 10.59571, + "29": 10.5743, + "30": 10.5518, + "31": 10.55277, + "32": 10.52963, "33": 10.49301, - "34": 10.46574, - "35": 10.46781, - "36": 10.4343, - "37": 10.40562, - "38": 10.40708, - "39": 10.38088, - "40": 10.35374, - "41": 10.33103, - "42": 10.31722, - "43": 10.27722, - "44": 10.27454, + "34": 10.46588, + "35": 10.46778, + "36": 10.43436, + "37": 10.40571, + "38": 10.4071, + "39": 10.3809, + "40": 10.35377, + "41": 10.3311, + "42": 10.3172, + "43": 10.27725, + "44": 10.27456, "45": 10.27064, - "46": 10.23923, - "47": 10.22013, - "48": 10.17386, - "49": 10.18283, - "50": 10.18243, - "51": 10.17726, - "52": 10.12542, - "53": 10.11928, - "54": 10.09855, - "55": 10.06505, + "46": 10.23931, + "47": 10.22016, + "48": 10.17388, + "49": 10.18285, + "50": 10.18249, + "51": 10.1773, + "52": 10.12547, + "53": 10.11934, + "54": 10.0986, + "55": 10.06513, "56": 10.096, - "57": 10.07028, - "58": 10.08099, - "59": 10.02378, - "60": 10.04446, - "61": 10.00535, - "62": 9.96184, - "63": 10.04441, - "64": 9.98732, - "65": 9.94123, - "66": 9.99568, - "67": 9.96328, - "68": 9.92304, - "69": 9.92538, - "70": 9.91447, - "71": 9.94788, - "72": 9.91509, + "57": 10.07031, + "58": 10.08104, + "59": 10.02385, + "60": 10.04452, + "61": 10.00537, + "62": 9.96197, + "63": 10.04444, + "64": 9.98734, + "65": 9.94133, + "66": 9.99575, + "67": 9.9634, + "68": 9.9231, + "69": 9.92541, + "70": 9.91453, + "71": 9.94791, + "72": 9.91515, "73": 9.89097, - "74": 9.8886, - "75": 9.84715, - "76": 9.8959, - "77": 9.89051, - "78": 9.84141, - "79": 9.83907, - "80": 9.85622, - "81": 9.88218, - "82": 9.83193, - "83": 9.77675, - "84": 9.71176, - "85": 9.69855, - "86": 9.81118, - "87": 9.84717, - "88": 9.81948, - "89": 9.74334, - "90": 9.74338, - "91": 9.75546, - "92": 9.75109, - "93": 9.66382, - "94": 9.7469, - "95": 9.74363, - "96": 9.7211, - "97": 9.65221, - "98": 9.68133, - "99": 9.75593, - "100": 9.62787 + "74": 9.88869, + "75": 9.84723, + "76": 9.89598, + "77": 9.89061, + "78": 9.84146, + "79": 9.83915, + "80": 9.85629, + "81": 9.88221, + "82": 9.83194, + "83": 9.77679, + "84": 9.71183, + "85": 9.69863, + "86": 9.81127, + "87": 9.84721, + "88": 9.81959, + "89": 9.74338, + "90": 9.74347, + "91": 9.75549, + "92": 9.75117, + "93": 9.66392, + "94": 9.74692, + "95": 9.7437, + "96": 9.72111, + "97": 9.65225, + "98": 9.68137, + "99": 9.75599, + "100": 9.62793 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1958.0, - "2": 1885.0, - "3": 1969.0, - "4": 1999.0, - "5": 1867.0, - "6": 1911.0, - "7": 2177.0, - "8": 1891.0, - "9": 1929.0, - "10": 1805.0, - "11": 1962.0, - "12": 1807.0, - "13": 2025.0, - "14": 2027.0, - "15": 1879.0, - "16": 1843.0, - "17": 1905.0, - "18": 1951.0, - "19": 1818.0, - "20": 1855.0, - "21": 2073.0, - "22": 1999.0, - "23": 1931.0, - "24": 1954.0, - "25": 1885.0, - "26": 1873.0, - "27": 1933.0, - "28": 1849.0, - "29": 1846.0, - "30": 1938.0, - "31": 2100.0, - "32": 1916.0, - "33": 1980.0, + "1": 1892.0, + "2": 1813.0, + "3": 1829.0, + "4": 1973.0, + "5": 1882.0, + "6": 2001.0, + "7": 2094.0, + "8": 1871.0, + "9": 1941.0, + "10": 1883.0, + "11": 1940.0, + "12": 1831.0, + "13": 1993.0, + "14": 1894.0, + "15": 1719.0, + "16": 1863.0, + "17": 1892.0, + "18": 1879.0, + "19": 1863.0, + "20": 1794.0, + "21": 2053.0, + "22": 1956.0, + "23": 1953.0, + "24": 2021.0, + "25": 1899.0, + "26": 2001.0, + "27": 2069.0, + "28": 1873.0, + "29": 1915.0, + "30": 1820.0, + "31": 2063.0, + "32": 1895.0, + "33": 1945.0, "34": 2037.0, - "35": 2186.0, - "36": 2150.0, - "37": 2263.0, - "38": 2156.0, - "39": 2189.0, - "40": 2404.0, - "41": 2412.0, - "42": 2103.0, - "43": 2402.0, - "44": 2177.0, - "45": 2409.0, - "46": 2603.0, - "47": 2426.0, - "48": 2473.0, - "49": 2667.0, - "50": 2655.0, - "51": 2533.0, - "52": 2515.0, - "53": 2545.0, - "54": 2733.0, - "55": 2493.0, - "56": 2587.0, - "57": 2363.0, - "58": 3251.0, - "59": 2811.0, - "60": 2742.0, - "61": 2684.0, - "62": 3000.0, - "63": 3044.0, - "64": 3028.0, - "65": 2515.0, - "66": 2860.0, - "67": 3289.0, - "68": 2968.0, - "69": 2906.0, - "70": 2961.0, - "71": 2931.0, - "72": 3076.0, - "73": 3177.0, - "74": 3036.0, - "75": 2891.0, - "76": 3299.0, - "77": 3251.0, - "78": 2998.0, - "79": 2895.0, - "80": 3002.0, - "81": 3079.0, - "82": 3229.0, - "83": 2912.0, - "84": 2982.0, - "85": 2899.0, - "86": 3206.0, - "87": 2993.0, - "88": 3126.0, - "89": 3111.0, - "90": 3471.0, - "91": 2851.0, - "92": 2966.0, - "93": 2875.0, - "94": 3261.0, - "95": 3139.0, - "96": 3289.0, - "97": 3318.0, - "98": 3249.0, - "99": 3044.0, - "100": 2947.0 + "35": 2099.0, + "36": 2164.0, + "37": 2249.0, + "38": 2197.0, + "39": 2095.0, + "40": 2400.0, + "41": 2421.0, + "42": 2180.0, + "43": 2362.0, + "44": 2254.0, + "45": 2458.0, + "46": 2484.0, + "47": 2365.0, + "48": 2506.0, + "49": 2664.0, + "50": 2618.0, + "51": 2408.0, + "52": 2520.0, + "53": 2461.0, + "54": 2709.0, + "55": 2468.0, + "56": 2650.0, + "57": 2388.0, + "58": 3177.0, + "59": 2817.0, + "60": 2841.0, + "61": 2659.0, + "62": 2852.0, + "63": 3055.0, + "64": 3053.0, + "65": 2545.0, + "66": 2937.0, + "67": 3209.0, + "68": 2972.0, + "69": 2964.0, + "70": 2833.0, + "71": 2937.0, + "72": 3097.0, + "73": 3281.0, + "74": 2983.0, + "75": 2940.0, + "76": 3261.0, + "77": 3204.0, + "78": 2954.0, + "79": 2999.0, + "80": 2958.0, + "81": 3106.0, + "82": 3079.0, + "83": 2905.0, + "84": 2938.0, + "85": 3005.0, + "86": 3324.0, + "87": 2949.0, + "88": 3234.0, + "89": 3061.0, + "90": 3457.0, + "91": 2908.0, + "92": 3059.0, + "93": 2999.0, + "94": 3250.0, + "95": 3091.0, + "96": 3211.0, + "97": 3342.0, + "98": 3247.0, + "99": 3091.0, + "100": 2944.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.48074, - "3": 0.11631, - "4": 0.11636, - "5": 0.11519, - "6": 0.11496, - "7": 0.11617, - "8": 0.11552, - "9": 0.11484, - "10": 0.11517, - "11": 0.11614, - "12": 0.11662, - "13": 0.11445, - "14": 0.11535, - "15": 0.11445, - "16": 0.11409, - "17": 0.11458, - "18": 0.11383, - "19": 0.11873, - "20": 0.11503, - "21": 0.11544, - "22": 0.1154, - "23": 0.1164, - "24": 0.12656, - "25": 0.11763, - "26": 0.11572, - "27": 0.11553, - "28": 0.1148, - "29": 0.11577, - "30": 0.11507, - "31": 0.11538, - "32": 0.11684, - "33": 0.1163, - "34": 0.1153, - "35": 0.1148, - "36": 0.11434, - "37": 0.11776, - "38": 0.11635, - "39": 0.11641, - "40": 0.1155, - "41": 0.1141, - "42": 0.11503, - "43": 0.11715, - "44": 0.11489, - "45": 0.11544, - "46": 0.11474, - "47": 0.11803, - "48": 0.11512, - "49": 0.11465, - "50": 0.11459, - "51": 0.14805, - "52": 0.1391, - "53": 0.12179, - "54": 0.12138, - "55": 0.11726, - "56": 0.11668, - "57": 0.11745, - "58": 0.11896, - "59": 0.11676, - "60": 0.11642, - "61": 0.11523, - "62": 0.11424, - "63": 0.11435, - "64": 0.11653, - "65": 0.11928, - "66": 0.11731, - "67": 0.11703, - "68": 0.11679, - "69": 0.11634, - "70": 0.11642, - "71": 0.11527, - "72": 0.11515, - "73": 0.1164, - "74": 0.11599, - "75": 0.11673, - "76": 0.11705, - "77": 0.11642, - "78": 0.11446, - "79": 0.11517, - "80": 0.11603, - "81": 0.11937, - "82": 0.11678, - "83": 0.11615, - "84": 0.11651, - "85": 0.11591, - "86": 0.11576, - "87": 0.11657, - "88": 0.11645, - "89": 0.11605, - "90": 0.11536, - "91": 0.11542, - "92": 0.11782, - "93": 0.1154, - "94": 0.11637, - "95": 0.11553, - "96": 0.11553, - "97": 0.11617, - "98": 0.11806, - "99": 0.11598, - "100": 0.11537 + "2": 4.49042, + "3": 0.12025, + "4": 0.1189, + "5": 0.11738, + "6": 0.11596, + "7": 0.11779, + "8": 0.11788, + "9": 0.11575, + "10": 0.11631, + "11": 0.11687, + "12": 0.12025, + "13": 0.11692, + "14": 0.11822, + "15": 0.11902, + "16": 0.11784, + "17": 0.11643, + "18": 0.11516, + "19": 0.11736, + "20": 0.12073, + "21": 0.11911, + "22": 0.12155, + "23": 0.11629, + "24": 0.11589, + "25": 0.11626, + "26": 0.11645, + "27": 0.11698, + "28": 0.11757, + "29": 0.11885, + "30": 0.11836, + "31": 0.11715, + "32": 0.11799, + "33": 0.1183, + "34": 0.118, + "35": 0.11637, + "36": 0.11748, + "37": 0.11656, + "38": 0.1167, + "39": 0.11714, + "40": 0.11928, + "41": 0.11629, + "42": 0.11738, + "43": 0.11826, + "44": 0.11668, + "45": 0.11767, + "46": 0.11588, + "47": 0.11571, + "48": 0.11982, + "49": 0.11749, + "50": 0.11873, + "51": 0.14907, + "52": 0.15187, + "53": 0.12628, + "54": 0.12237, + "55": 0.12294, + "56": 0.12255, + "57": 0.12154, + "58": 0.12203, + "59": 0.12091, + "60": 0.11978, + "61": 0.12078, + "62": 0.12075, + "63": 0.11931, + "64": 0.1171, + "65": 0.11919, + "66": 0.12149, + "67": 0.11999, + "68": 0.1213, + "69": 0.11895, + "70": 0.11891, + "71": 0.11884, + "72": 0.11978, + "73": 0.12015, + "74": 0.12009, + "75": 0.12143, + "76": 0.11883, + "77": 0.11985, + "78": 0.11844, + "79": 0.11749, + "80": 0.1186, + "81": 0.11856, + "82": 0.11958, + "83": 0.11774, + "84": 0.11977, + "85": 0.11701, + "86": 0.11746, + "87": 0.11884, + "88": 0.11768, + "89": 0.1175, + "90": 0.11798, + "91": 0.11699, + "92": 0.11867, + "93": 0.11883, + "94": 0.11892, + "95": 0.11809, + "96": 0.11972, + "97": 0.11759, + "98": 0.11778, + "99": 0.1178, + "100": 0.11902 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json index ecd3d1ee35c..65546a0734c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90185, + "1": 10.90187, "2": 10.90476, - "3": 10.91298, - "4": 10.90732, - "5": 10.90338, - "6": 10.88352, - "7": 10.89466, - "8": 10.9003, - "9": 10.90241, - "10": 10.90124, - "11": 10.89365, - "12": 10.88403, - "13": 10.87658, - "14": 10.87083, - "15": 10.86053, - "16": 10.85047, - "17": 10.85366, - "18": 10.83505, - "19": 10.83504, - "20": 10.75556, - "21": 10.75273, - "22": 10.74048, - "23": 10.73225, - "24": 10.6904, - "25": 10.70403, - "26": 10.68252, - "27": 10.64963, - "28": 10.5671, - "29": 10.54339, - "30": 10.51454, - "31": 10.51312, + "3": 10.91307, + "4": 10.90729, + "5": 10.90346, + "6": 10.88351, + "7": 10.89468, + "8": 10.90036, + "9": 10.90242, + "10": 10.90119, + "11": 10.89375, + "12": 10.8841, + "13": 10.8767, + "14": 10.87086, + "15": 10.86051, + "16": 10.85048, + "17": 10.85362, + "18": 10.83516, + "19": 10.83498, + "20": 10.75559, + "21": 10.75274, + "22": 10.74047, + "23": 10.73228, + "24": 10.69032, + "25": 10.70411, + "26": 10.68255, + "27": 10.64974, + "28": 10.56717, + "29": 10.54336, + "30": 10.5146, + "31": 10.51313, "32": 10.49085, - "33": 10.46425, - "34": 10.41243, - "35": 10.41921, - "36": 10.40599, - "37": 10.35423, - "38": 10.36112, - "39": 10.32243, - "40": 10.32058, - "41": 10.28482, - "42": 10.27525, - "43": 10.24427, - "44": 10.21985, + "33": 10.46423, + "34": 10.41249, + "35": 10.41923, + "36": 10.40607, + "37": 10.35428, + "38": 10.36119, + "39": 10.32248, + "40": 10.3206, + "41": 10.28486, + "42": 10.27528, + "43": 10.24433, + "44": 10.21989, "45": 10.23687, - "46": 10.19624, - "47": 10.18389, - "48": 10.1323, - "49": 10.13047, - "50": 10.14436, - "51": 10.14123, - "52": 10.09131, - "53": 10.08918, - "54": 10.06284, - "55": 10.01711, - "56": 10.07149, - "57": 10.03622, - "58": 10.06524, - "59": 10.00246, - "60": 10.01718, - "61": 9.98392, - "62": 9.93675, - "63": 10.03256, - "64": 9.96969, - "65": 9.93232, - "66": 9.97442, - "67": 9.94335, - "68": 9.8968, - "69": 9.9134, - "70": 9.9038, - "71": 9.93369, - "72": 9.89264, - "73": 9.87323, - "74": 9.87541, - "75": 9.84114, - "76": 9.89728, - "77": 9.8872, + "46": 10.19634, + "47": 10.18395, + "48": 10.13233, + "49": 10.13056, + "50": 10.14445, + "51": 10.14128, + "52": 10.09134, + "53": 10.08921, + "54": 10.06288, + "55": 10.01713, + "56": 10.07152, + "57": 10.0363, + "58": 10.06528, + "59": 10.0025, + "60": 10.0172, + "61": 9.98397, + "62": 9.93682, + "63": 10.03263, + "64": 9.9697, + "65": 9.9324, + "66": 9.97451, + "67": 9.94342, + "68": 9.89685, + "69": 9.91349, + "70": 9.90387, + "71": 9.93377, + "72": 9.89268, + "73": 9.87331, + "74": 9.87543, + "75": 9.84116, + "76": 9.89725, + "77": 9.88725, "78": 9.82611, - "79": 9.83323, - "80": 9.8524, - "81": 9.87595, - "82": 9.82806, - "83": 9.7717, - "84": 9.70894, - "85": 9.69359, - "86": 9.80476, - "87": 9.8523, - "88": 9.82792, - "89": 9.74787, - "90": 9.74092, - "91": 9.7535, - "92": 9.74928, - "93": 9.67428, - "94": 9.75078, - "95": 9.75704, - "96": 9.73734, - "97": 9.6548, + "79": 9.83326, + "80": 9.85244, + "81": 9.87596, + "82": 9.82808, + "83": 9.77177, + "84": 9.70902, + "85": 9.69361, + "86": 9.80481, + "87": 9.85232, + "88": 9.82795, + "89": 9.7479, + "90": 9.74098, + "91": 9.75355, + "92": 9.74932, + "93": 9.67432, + "94": 9.7508, + "95": 9.75706, + "96": 9.73746, + "97": 9.65486, "98": 9.69864, - "99": 9.75611, - "100": 9.63811 + "99": 9.75613, + "100": 9.63818 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1985.0, - "2": 1849.0, - "3": 1918.0, - "4": 1845.0, - "5": 1981.0, - "6": 1867.0, - "7": 2134.0, - "8": 1795.0, - "9": 1915.0, - "10": 2000.0, - "11": 1951.0, - "12": 1871.0, - "13": 1991.0, - "14": 2057.0, - "15": 1826.0, - "16": 1941.0, - "17": 2086.0, - "18": 2095.0, - "19": 2002.0, - "20": 1853.0, - "21": 2090.0, - "22": 1944.0, - "23": 1847.0, - "24": 2008.0, - "25": 1831.0, - "26": 2011.0, - "27": 1982.0, - "28": 1928.0, - "29": 2119.0, - "30": 1923.0, - "31": 2072.0, - "32": 1955.0, - "33": 2018.0, - "34": 2088.0, - "35": 2066.0, - "36": 2037.0, - "37": 2227.0, - "38": 2208.0, - "39": 2078.0, - "40": 2359.0, - "41": 2345.0, - "42": 2083.0, - "43": 2417.0, - "44": 2281.0, - "45": 2524.0, - "46": 2404.0, - "47": 2372.0, - "48": 2558.0, - "49": 2718.0, - "50": 2502.0, - "51": 2528.0, - "52": 2618.0, - "53": 2473.0, - "54": 2778.0, - "55": 2518.0, - "56": 2819.0, - "57": 2354.0, - "58": 3243.0, - "59": 2808.0, - "60": 2877.0, - "61": 2627.0, - "62": 2989.0, - "63": 3013.0, - "64": 3270.0, - "65": 2533.0, - "66": 2795.0, - "67": 3328.0, - "68": 3128.0, - "69": 3087.0, - "70": 2901.0, - "71": 3034.0, - "72": 3116.0, - "73": 3270.0, - "74": 3013.0, - "75": 3097.0, - "76": 3258.0, - "77": 3142.0, - "78": 3204.0, + "1": 2015.0, + "2": 1861.0, + "3": 1941.0, + "4": 1935.0, + "5": 1965.0, + "6": 1843.0, + "7": 2203.0, + "8": 1887.0, + "9": 1903.0, + "10": 2052.0, + "11": 2028.0, + "12": 1917.0, + "13": 2102.0, + "14": 2083.0, + "15": 1736.0, + "16": 1963.0, + "17": 2051.0, + "18": 1995.0, + "19": 1887.0, + "20": 1908.0, + "21": 2013.0, + "22": 2004.0, + "23": 1792.0, + "24": 1939.0, + "25": 1931.0, + "26": 2003.0, + "27": 1989.0, + "28": 1899.0, + "29": 2110.0, + "30": 1874.0, + "31": 2037.0, + "32": 1981.0, + "33": 1993.0, + "34": 2156.0, + "35": 2084.0, + "36": 2097.0, + "37": 2228.0, + "38": 2216.0, + "39": 2142.0, + "40": 2334.0, + "41": 2302.0, + "42": 2107.0, + "43": 2369.0, + "44": 2243.0, + "45": 2546.0, + "46": 2378.0, + "47": 2433.0, + "48": 2559.0, + "49": 2660.0, + "50": 2677.0, + "51": 2560.0, + "52": 2678.0, + "53": 2549.0, + "54": 2732.0, + "55": 2532.0, + "56": 2783.0, + "57": 2378.0, + "58": 3179.0, + "59": 2762.0, + "60": 2809.0, + "61": 2644.0, + "62": 2957.0, + "63": 3040.0, + "64": 3145.0, + "65": 2674.0, + "66": 2825.0, + "67": 3388.0, + "68": 3169.0, + "69": 3036.0, + "70": 2791.0, + "71": 2990.0, + "72": 3030.0, + "73": 3211.0, + "74": 3126.0, + "75": 3025.0, + "76": 3267.0, + "77": 3317.0, + "78": 3256.0, "79": 2963.0, - "80": 3126.0, - "81": 3222.0, - "82": 3247.0, - "83": 2893.0, - "84": 2910.0, - "85": 2985.0, - "86": 3218.0, - "87": 3275.0, - "88": 3179.0, - "89": 3082.0, - "90": 3491.0, - "91": 2820.0, - "92": 2940.0, - "93": 2944.0, - "94": 3371.0, - "95": 3239.0, - "96": 3199.0, - "97": 3432.0, - "98": 3457.0, - "99": 3228.0, - "100": 3028.0 + "80": 3088.0, + "81": 3243.0, + "82": 3238.0, + "83": 3035.0, + "84": 2916.0, + "85": 2879.0, + "86": 3213.0, + "87": 3332.0, + "88": 3177.0, + "89": 2975.0, + "90": 3504.0, + "91": 2955.0, + "92": 3004.0, + "93": 3022.0, + "94": 3272.0, + "95": 3238.0, + "96": 3220.0, + "97": 3279.0, + "98": 3385.0, + "99": 3139.0, + "100": 2992.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.52139, - "3": 1.31676, - "4": 0.79975, - "5": 0.76142, - "6": 0.61688, - "7": 1.10994, - "8": 1.21103, - "9": 1.26967, - "10": 0.80736, - "11": 1.03133, - "12": 1.01682, - "13": 1.14165, - "14": 1.17872, - "15": 1.03174, - "16": 0.98198, - "17": 1.20988, - "18": 1.13469, - "19": 0.8773, - "20": 0.87974, - "21": 0.83743, - "22": 1.10107, - "23": 1.1823, - "24": 1.44275, - "25": 0.63836, - "26": 1.00997, - "27": 1.08467, - "28": 1.30864, - "29": 0.74933, - "30": 1.0344, - "31": 0.73649, - "32": 1.78843, - "33": 1.18019, - "34": 0.3539, - "35": 1.39269, - "36": 1.12814, - "37": 0.87369, - "38": 1.17387, - "39": 1.43797, - "40": 1.61001, - "41": 1.11028, - "42": 1.17756, - "43": 0.87467, - "44": 1.52143, - "45": 1.04728, - "46": 0.87365, - "47": 1.2417, - "48": 0.95874, - "49": 0.83764, - "50": 0.97516, - "51": 0.50004, - "52": 1.16182, - "53": 1.048, - "54": 1.25479, - "55": 0.92458, - "56": 0.9998, - "57": 0.87311, - "58": 0.81221, - "59": 0.94875, - "60": 1.04215, - "61": 0.85544, - "62": 1.22979, - "63": 0.7709, - "64": 1.2745, - "65": 0.72474, - "66": 1.08558, - "67": 1.23927, - "68": 0.63327, - "69": 1.35395, - "70": 1.03255, - "71": 0.96629, - "72": 1.06889, - "73": 0.98471, - "74": 0.58585, - "75": 0.88547, - "76": 0.98341, - "77": 0.94806, - "78": 1.08725, - "79": 1.90974, - "80": 1.09005, - "81": 0.91725, - "82": 1.27878, - "83": 1.01817, - "84": 0.80636, - "85": 0.45266, - "86": 0.67137, - "87": 1.50911, - "88": 0.69126, - "89": 0.54835, - "90": 1.20966, - "91": 0.86093, - "92": 1.26522, - "93": 0.90824, - "94": 1.01004, - "95": 0.76243, - "96": 1.13206, - "97": 1.03872, - "98": 1.08888, - "99": 1.4323, - "100": 1.16297 + "2": 3.91019, + "3": 1.14853, + "4": 0.91021, + "5": 0.4773, + "6": 1.14396, + "7": 1.02475, + "8": 1.32173, + "9": 1.37987, + "10": 0.87461, + "11": 1.00485, + "12": 0.97421, + "13": 1.07136, + "14": 0.88819, + "15": 0.76849, + "16": 0.9084, + "17": 1.15647, + "18": 0.98663, + "19": 0.73933, + "20": 0.94501, + "21": 1.07236, + "22": 1.00681, + "23": 1.06235, + "24": 1.15474, + "25": 0.72107, + "26": 0.85786, + "27": 1.10168, + "28": 0.93285, + "29": 0.94744, + "30": 0.82371, + "31": 0.74059, + "32": 1.48375, + "33": 1.13609, + "34": 0.41336, + "35": 1.34094, + "36": 1.29471, + "37": 0.96383, + "38": 0.68931, + "39": 0.85504, + "40": 1.27906, + "41": 0.99075, + "42": 1.1342, + "43": 0.72673, + "44": 1.358, + "45": 1.07936, + "46": 0.91372, + "47": 1.09874, + "48": 1.02794, + "49": 0.94732, + "50": 1.19209, + "51": 0.62755, + "52": 0.98559, + "53": 0.99083, + "54": 1.02728, + "55": 0.89677, + "56": 1.26438, + "57": 0.85156, + "58": 0.92718, + "59": 0.6928, + "60": 0.57307, + "61": 1.21843, + "62": 0.93413, + "63": 0.71419, + "64": 1.18317, + "65": 0.66956, + "66": 1.29254, + "67": 1.28158, + "68": 0.86434, + "69": 1.23377, + "70": 0.94656, + "71": 1.0478, + "72": 1.01223, + "73": 0.96642, + "74": 0.6654, + "75": 0.83866, + "76": 1.00455, + "77": 1.07311, + "78": 1.05199, + "79": 1.61781, + "80": 1.16326, + "81": 0.79197, + "82": 1.48728, + "83": 1.22728, + "84": 0.98578, + "85": 0.71056, + "86": 0.87975, + "87": 1.70621, + "88": 0.96303, + "89": 0.71006, + "90": 1.41135, + "91": 0.83753, + "92": 1.02212, + "93": 1.05654, + "94": 1.4589, + "95": 0.81371, + "96": 1.02002, + "97": 0.90401, + "98": 0.68534, + "99": 1.4266, + "100": 0.99165 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json index 9ae0664b7b0..2eac544b91f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96078, - "2": 10.94368, - "3": 10.93755, - "4": 10.95045, - "5": 10.94855, - "6": 10.94271, - "7": 10.93509, - "8": 10.94175, - "9": 10.9418, - "10": 10.92772, - "11": 10.93415, - "12": 10.94481, - "13": 10.92208, - "14": 10.92272, - "15": 10.89966, + "1": 10.96081, + "2": 10.94369, + "3": 10.93753, + "4": 10.9505, + "5": 10.94843, + "6": 10.94269, + "7": 10.93505, + "8": 10.9418, + "9": 10.94177, + "10": 10.9278, + "11": 10.93407, + "12": 10.94483, + "13": 10.9221, + "14": 10.92271, + "15": 10.89965, "16": 10.89041, - "17": 10.88898, - "18": 10.88212, - "19": 10.87681, - "20": 10.80091, - "21": 10.7814, - "22": 10.76174, - "23": 10.76213, - "24": 10.73664, - "25": 10.73447, - "26": 10.72101, - "27": 10.68102, - "28": 10.59583, - "29": 10.57443, - "30": 10.55201, - "31": 10.55316, - "32": 10.52965, + "17": 10.88902, + "18": 10.88214, + "19": 10.87673, + "20": 10.801, + "21": 10.78143, + "22": 10.76172, + "23": 10.76224, + "24": 10.73665, + "25": 10.73451, + "26": 10.72103, + "27": 10.68111, + "28": 10.59594, + "29": 10.57444, + "30": 10.55211, + "31": 10.55324, + "32": 10.52967, "33": 10.49304, - "34": 10.46592, - "35": 10.46781, - "36": 10.43445, - "37": 10.40557, - "38": 10.40698, - "39": 10.38092, - "40": 10.35372, - "41": 10.33113, - "42": 10.31728, - "43": 10.27747, - "44": 10.27475, - "45": 10.27088, - "46": 10.2394, - "47": 10.22002, - "48": 10.17409, - "49": 10.18283, - "50": 10.18254, - "51": 10.17736, - "52": 10.12545, - "53": 10.11938, - "54": 10.09849, - "55": 10.06505, - "56": 10.09606, - "57": 10.0703, - "58": 10.08109, - "59": 10.02394, - "60": 10.04465, - "61": 10.00552, - "62": 9.96193, - "63": 10.0445, - "64": 9.98729, - "65": 9.94156, - "66": 9.99581, - "67": 9.96348, - "68": 9.92327, - "69": 9.92556, - "70": 9.91456, - "71": 9.94802, - "72": 9.91519, - "73": 9.89109, - "74": 9.88876, - "75": 9.84728, - "76": 9.896, - "77": 9.89059, - "78": 9.84155, - "79": 9.83898, - "80": 9.85635, - "81": 9.88226, - "82": 9.83206, - "83": 9.77654, - "84": 9.71211, - "85": 9.69857, - "86": 9.81127, - "87": 9.84704, - "88": 9.81952, - "89": 9.74322, - "90": 9.74316, - "91": 9.75539, - "92": 9.75092, - "93": 9.66352, - "94": 9.74704, - "95": 9.74356, - "96": 9.72114, - "97": 9.65242, - "98": 9.68127, - "99": 9.75572, - "100": 9.62796 + "34": 10.46601, + "35": 10.46785, + "36": 10.43451, + "37": 10.40567, + "38": 10.40699, + "39": 10.38093, + "40": 10.35377, + "41": 10.33117, + "42": 10.31723, + "43": 10.27752, + "44": 10.27479, + "45": 10.27097, + "46": 10.23942, + "47": 10.2201, + "48": 10.17413, + "49": 10.18288, + "50": 10.18261, + "51": 10.17743, + "52": 10.1255, + "53": 10.1194, + "54": 10.09854, + "55": 10.06517, + "56": 10.09613, + "57": 10.07035, + "58": 10.08113, + "59": 10.02402, + "60": 10.04469, + "61": 10.00553, + "62": 9.96201, + "63": 10.04454, + "64": 9.98737, + "65": 9.94162, + "66": 9.99585, + "67": 9.96358, + "68": 9.92335, + "69": 9.9256, + "70": 9.91461, + "71": 9.94806, + "72": 9.91522, + "73": 9.89107, + "74": 9.88884, + "75": 9.84737, + "76": 9.89606, + "77": 9.89063, + "78": 9.84159, + "79": 9.83907, + "80": 9.85636, + "81": 9.8823, + "82": 9.83207, + "83": 9.77662, + "84": 9.71215, + "85": 9.69867, + "86": 9.81131, + "87": 9.84706, + "88": 9.81954, + "89": 9.74321, + "90": 9.74319, + "91": 9.75545, + "92": 9.751, + "93": 9.6636, + "94": 9.74709, + "95": 9.74364, + "96": 9.72117, + "97": 9.65251, + "98": 9.68136, + "99": 9.75576, + "100": 9.62804 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1951.0, - "2": 1836.0, - "3": 1829.0, - "4": 1977.0, - "5": 1855.0, - "6": 1949.0, - "7": 2113.0, - "8": 1918.0, - "9": 1917.0, - "10": 1879.0, - "11": 1898.0, - "12": 1857.0, - "13": 1993.0, - "14": 1891.0, - "15": 1785.0, - "16": 1893.0, - "17": 1913.0, - "18": 1905.0, - "19": 1905.0, - "20": 1878.0, - "21": 1993.0, - "22": 1888.0, - "23": 1913.0, - "24": 2001.0, - "25": 1886.0, - "26": 1952.0, - "27": 2053.0, - "28": 1923.0, - "29": 1995.0, - "30": 1986.0, - "31": 2097.0, - "32": 1946.0, - "33": 1942.0, - "34": 2090.0, - "35": 2120.0, - "36": 2074.0, - "37": 2211.0, - "38": 2238.0, - "39": 2252.0, - "40": 2414.0, - "41": 2320.0, - "42": 2143.0, - "43": 2265.0, - "44": 2227.0, - "45": 2362.0, - "46": 2460.0, - "47": 2462.0, - "48": 2501.0, - "49": 2790.0, - "50": 2607.0, - "51": 2561.0, - "52": 2482.0, - "53": 2547.0, - "54": 2803.0, - "55": 2441.0, - "56": 2657.0, - "57": 2351.0, - "58": 3218.0, - "59": 2723.0, - "60": 2830.0, - "61": 2533.0, - "62": 2975.0, - "63": 3074.0, - "64": 3077.0, - "65": 2643.0, - "66": 2867.0, - "67": 3265.0, - "68": 2940.0, - "69": 2946.0, - "70": 2812.0, - "71": 2922.0, - "72": 3002.0, - "73": 3131.0, - "74": 3011.0, - "75": 2877.0, - "76": 3099.0, - "77": 3165.0, - "78": 3083.0, - "79": 2875.0, - "80": 2910.0, - "81": 3101.0, - "82": 3122.0, - "83": 2955.0, - "84": 2874.0, - "85": 2745.0, - "86": 3208.0, - "87": 3043.0, - "88": 3223.0, - "89": 3100.0, - "90": 3461.0, - "91": 2883.0, - "92": 3192.0, - "93": 2895.0, - "94": 3337.0, - "95": 3129.0, - "96": 3285.0, - "97": 3254.0, - "98": 3248.0, - "99": 2989.0, - "100": 3047.0 + "1": 1992.0, + "2": 1841.0, + "3": 1914.0, + "4": 1948.0, + "5": 1885.0, + "6": 1882.0, + "7": 2104.0, + "8": 1794.0, + "9": 2000.0, + "10": 1885.0, + "11": 1969.0, + "12": 1822.0, + "13": 1976.0, + "14": 1912.0, + "15": 1777.0, + "16": 1918.0, + "17": 1910.0, + "18": 1975.0, + "19": 1918.0, + "20": 1814.0, + "21": 2015.0, + "22": 1917.0, + "23": 1851.0, + "24": 1993.0, + "25": 1942.0, + "26": 1890.0, + "27": 2014.0, + "28": 1874.0, + "29": 1920.0, + "30": 1933.0, + "31": 2018.0, + "32": 2008.0, + "33": 1946.0, + "34": 2065.0, + "35": 2135.0, + "36": 2072.0, + "37": 2212.0, + "38": 2241.0, + "39": 2200.0, + "40": 2444.0, + "41": 2309.0, + "42": 2099.0, + "43": 2373.0, + "44": 2276.0, + "45": 2524.0, + "46": 2509.0, + "47": 2404.0, + "48": 2468.0, + "49": 2814.0, + "50": 2574.0, + "51": 2497.0, + "52": 2399.0, + "53": 2556.0, + "54": 2739.0, + "55": 2386.0, + "56": 2640.0, + "57": 2399.0, + "58": 3307.0, + "59": 2740.0, + "60": 2792.0, + "61": 2740.0, + "62": 2861.0, + "63": 3123.0, + "64": 3132.0, + "65": 2624.0, + "66": 2892.0, + "67": 3351.0, + "68": 3085.0, + "69": 2930.0, + "70": 2914.0, + "71": 2933.0, + "72": 3119.0, + "73": 3123.0, + "74": 3019.0, + "75": 2916.0, + "76": 3189.0, + "77": 3192.0, + "78": 2984.0, + "79": 2871.0, + "80": 2980.0, + "81": 3099.0, + "82": 3032.0, + "83": 2914.0, + "84": 3029.0, + "85": 2896.0, + "86": 3192.0, + "87": 2988.0, + "88": 3174.0, + "89": 3025.0, + "90": 3406.0, + "91": 2881.0, + "92": 3026.0, + "93": 2989.0, + "94": 3342.0, + "95": 3193.0, + "96": 3319.0, + "97": 3268.0, + "98": 3211.0, + "99": 3120.0, + "100": 2925.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 919557120.0, - "2": 919557120.0, - "3": 919557120.0, - "4": 919557120.0, - "5": 919557120.0, - "6": 919557120.0, - "7": 919557120.0, - "8": 919557120.0, - "9": 919557120.0, - "10": 919557120.0, - "11": 919557120.0, - "12": 919557120.0, - "13": 919557120.0, - "14": 919557120.0, - "15": 919557120.0, - "16": 919557120.0, - "17": 919557120.0, - "18": 919557120.0, - "19": 919557120.0, - "20": 919557120.0, - "21": 919557120.0, - "22": 919557120.0, - "23": 919557120.0, - "24": 919557120.0, - "25": 919557120.0, - "26": 919557120.0, - "27": 919557120.0, - "28": 919557120.0, - "29": 919557120.0, - "30": 919557120.0, - "31": 919557120.0, - "32": 919557120.0, - "33": 919557120.0, - "34": 919557120.0, - "35": 919557120.0, - "36": 919557120.0, - "37": 919557120.0, - "38": 919557120.0, - "39": 919557120.0, - "40": 919557120.0, - "41": 919557120.0, - "42": 919557120.0, - "43": 919557120.0, - "44": 919557120.0, - "45": 919557120.0, - "46": 919557120.0, - "47": 919557120.0, - "48": 919557120.0, - "49": 919557120.0, - "50": 919557120.0, - "51": 919557120.0, - "52": 919557120.0, - "53": 919557120.0, - "54": 919557120.0, - "55": 919557120.0, - "56": 919557120.0, - "57": 919557120.0, - "58": 919557120.0, - "59": 919557120.0, - "60": 919557120.0, - "61": 919557120.0, - "62": 919557120.0, - "63": 919557120.0, - "64": 919557120.0, - "65": 919557120.0, - "66": 919557120.0, - "67": 919557120.0, - "68": 919557120.0, - "69": 919557120.0, - "70": 919557120.0, - "71": 919557120.0, - "72": 919557120.0, - "73": 919557120.0, - "74": 919557120.0, - "75": 919557120.0, - "76": 919557120.0, - "77": 919557120.0, - "78": 919557120.0, - "79": 919557120.0, - "80": 919557120.0, - "81": 919557120.0, - "82": 919557120.0, - "83": 919557120.0, - "84": 919557120.0, - "85": 919557120.0, - "86": 919557120.0, - "87": 919557120.0, - "88": 919557120.0, - "89": 919557120.0, - "90": 919557120.0, - "91": 919557120.0, - "92": 919557120.0, - "93": 919557120.0, - "94": 919557120.0, - "95": 919557120.0, - "96": 919557120.0, - "97": 919557120.0, - "98": 919557120.0, - "99": 919557120.0, - "100": 919557120.0 + "1": 917459968.0, + "2": 917459968.0, + "3": 917459968.0, + "4": 917459968.0, + "5": 917459968.0, + "6": 917459968.0, + "7": 917459968.0, + "8": 917459968.0, + "9": 917459968.0, + "10": 917459968.0, + "11": 917459968.0, + "12": 917459968.0, + "13": 917459968.0, + "14": 917459968.0, + "15": 917459968.0, + "16": 917459968.0, + "17": 917459968.0, + "18": 917459968.0, + "19": 917459968.0, + "20": 917459968.0, + "21": 917459968.0, + "22": 917459968.0, + "23": 917459968.0, + "24": 917459968.0, + "25": 917459968.0, + "26": 917459968.0, + "27": 917459968.0, + "28": 917459968.0, + "29": 917459968.0, + "30": 917459968.0, + "31": 917459968.0, + "32": 917459968.0, + "33": 917459968.0, + "34": 917459968.0, + "35": 917459968.0, + "36": 917459968.0, + "37": 917459968.0, + "38": 917459968.0, + "39": 917459968.0, + "40": 917459968.0, + "41": 917459968.0, + "42": 917459968.0, + "43": 917459968.0, + "44": 917459968.0, + "45": 917459968.0, + "46": 917459968.0, + "47": 917459968.0, + "48": 917459968.0, + "49": 917459968.0, + "50": 917459968.0, + "51": 917459968.0, + "52": 917459968.0, + "53": 917459968.0, + "54": 917459968.0, + "55": 917459968.0, + "56": 917459968.0, + "57": 917459968.0, + "58": 917459968.0, + "59": 917459968.0, + "60": 917459968.0, + "61": 917459968.0, + "62": 917459968.0, + "63": 917459968.0, + "64": 917459968.0, + "65": 917459968.0, + "66": 917459968.0, + "67": 917459968.0, + "68": 917459968.0, + "69": 917459968.0, + "70": 917459968.0, + "71": 917459968.0, + "72": 917459968.0, + "73": 917459968.0, + "74": 917459968.0, + "75": 917459968.0, + "76": 917459968.0, + "77": 917459968.0, + "78": 917459968.0, + "79": 917459968.0, + "80": 917459968.0, + "81": 917459968.0, + "82": 917459968.0, + "83": 917459968.0, + "84": 917459968.0, + "85": 917459968.0, + "86": 917459968.0, + "87": 917459968.0, + "88": 917459968.0, + "89": 917459968.0, + "90": 917459968.0, + "91": 917459968.0, + "92": 917459968.0, + "93": 917459968.0, + "94": 917459968.0, + "95": 917459968.0, + "96": 917459968.0, + "97": 917459968.0, + "98": 917459968.0, + "99": 917459968.0, + "100": 917459968.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2240869376.0, - "2": 2601383936.0, - "3": 2601383936.0, - "4": 2601383936.0, - "5": 2601383936.0, - "6": 2601383936.0, - "7": 2601383936.0, - "8": 2601383936.0, - "9": 2601383936.0, - "10": 2601383936.0, - "11": 2601383936.0, - "12": 2601383936.0, - "13": 2601383936.0, - "14": 2601383936.0, - "15": 2601383936.0, - "16": 2601383936.0, - "17": 2601383936.0, - "18": 2601383936.0, - "19": 2601383936.0, - "20": 2601383936.0, - "21": 2601383936.0, - "22": 2601383936.0, - "23": 2601383936.0, - "24": 2601383936.0, - "25": 2601383936.0, - "26": 2601383936.0, - "27": 2601383936.0, - "28": 2601383936.0, - "29": 2601383936.0, - "30": 2601383936.0, - "31": 2601383936.0, - "32": 2601383936.0, - "33": 2601383936.0, - "34": 2601383936.0, - "35": 2601383936.0, - "36": 2601383936.0, - "37": 2601383936.0, - "38": 2601383936.0, - "39": 2601383936.0, - "40": 2601383936.0, - "41": 2601383936.0, - "42": 2601383936.0, - "43": 2601383936.0, - "44": 2601383936.0, - "45": 2601383936.0, - "46": 2601383936.0, - "47": 2601383936.0, - "48": 2601383936.0, - "49": 2601383936.0, - "50": 2601383936.0, - "51": 2601383936.0, - "52": 2601383936.0, - "53": 2601383936.0, - "54": 2601383936.0, - "55": 2601383936.0, - "56": 2601383936.0, - "57": 2601383936.0, - "58": 2601383936.0, - "59": 2601383936.0, - "60": 2601383936.0, - "61": 2601383936.0, - "62": 2601383936.0, - "63": 2601383936.0, - "64": 2601383936.0, - "65": 2601383936.0, - "66": 2601383936.0, - "67": 2601383936.0, - "68": 2601383936.0, - "69": 2601383936.0, - "70": 2601383936.0, - "71": 2601383936.0, - "72": 2601383936.0, - "73": 2601383936.0, - "74": 2601383936.0, - "75": 2601383936.0, - "76": 2601383936.0, - "77": 2601383936.0, - "78": 2601383936.0, - "79": 2601383936.0, - "80": 2601383936.0, - "81": 2601383936.0, - "82": 2601383936.0, - "83": 2601383936.0, - "84": 2601383936.0, - "85": 2601383936.0, - "86": 2601383936.0, - "87": 2601383936.0, - "88": 2601383936.0, - "89": 2601383936.0, - "90": 2601383936.0, - "91": 2601383936.0, - "92": 2601383936.0, - "93": 2601383936.0, - "94": 2601383936.0, - "95": 2601383936.0, - "96": 2601383936.0, - "97": 2601383936.0, - "98": 2601383936.0, - "99": 2601383936.0, - "100": 2601383936.0 + "2": 2599286784.0, + "3": 2599286784.0, + "4": 2599286784.0, + "5": 2599286784.0, + "6": 2599286784.0, + "7": 2599286784.0, + "8": 2599286784.0, + "9": 2599286784.0, + "10": 2599286784.0, + "11": 2599286784.0, + "12": 2599286784.0, + "13": 2599286784.0, + "14": 2599286784.0, + "15": 2599286784.0, + "16": 2599286784.0, + "17": 2599286784.0, + "18": 2599286784.0, + "19": 2599286784.0, + "20": 2599286784.0, + "21": 2599286784.0, + "22": 2599286784.0, + "23": 2599286784.0, + "24": 2599286784.0, + "25": 2599286784.0, + "26": 2599286784.0, + "27": 2599286784.0, + "28": 2599286784.0, + "29": 2599286784.0, + "30": 2599286784.0, + "31": 2599286784.0, + "32": 2599286784.0, + "33": 2599286784.0, + "34": 2599286784.0, + "35": 2599286784.0, + "36": 2599286784.0, + "37": 2599286784.0, + "38": 2599286784.0, + "39": 2599286784.0, + "40": 2599286784.0, + "41": 2599286784.0, + "42": 2599286784.0, + "43": 2599286784.0, + "44": 2599286784.0, + "45": 2599286784.0, + "46": 2599286784.0, + "47": 2599286784.0, + "48": 2599286784.0, + "49": 2599286784.0, + "50": 2599286784.0, + "51": 2599286784.0, + "52": 2599286784.0, + "53": 2599286784.0, + "54": 2599286784.0, + "55": 2599286784.0, + "56": 2599286784.0, + "57": 2599286784.0, + "58": 2599286784.0, + "59": 2599286784.0, + "60": 2599286784.0, + "61": 2599286784.0, + "62": 2599286784.0, + "63": 2599286784.0, + "64": 2599286784.0, + "65": 2599286784.0, + "66": 2599286784.0, + "67": 2599286784.0, + "68": 2599286784.0, + "69": 2599286784.0, + "70": 2599286784.0, + "71": 2599286784.0, + "72": 2599286784.0, + "73": 2599286784.0, + "74": 2599286784.0, + "75": 2599286784.0, + "76": 2599286784.0, + "77": 2599286784.0, + "78": 2599286784.0, + "79": 2599286784.0, + "80": 2599286784.0, + "81": 2599286784.0, + "82": 2599286784.0, + "83": 2599286784.0, + "84": 2599286784.0, + "85": 2599286784.0, + "86": 2599286784.0, + "87": 2599286784.0, + "88": 2599286784.0, + "89": 2599286784.0, + "90": 2599286784.0, + "91": 2599286784.0, + "92": 2599286784.0, + "93": 2599286784.0, + "94": 2599286784.0, + "95": 2599286784.0, + "96": 2599286784.0, + "97": 2599286784.0, + "98": 2599286784.0, + "99": 2599286784.0, + "100": 2599286784.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.79089, - "3": 0.12042, - "4": 1.33387, - "5": 0.42202, - "6": 0.74472, - "7": 1.03792, - "8": 1.10582, - "9": 0.69905, - "10": 0.35146, - "11": 0.98439, - "12": 0.75226, - "13": 0.47222, - "14": 0.81203, - "15": 0.58281, - "16": 0.59479, - "17": 0.58081, - "18": 1.02468, - "19": 0.32551, - "20": 0.67874, - "21": 0.74181, - "22": 0.68122, - "23": 0.60888, - "24": 0.78068, - "25": 0.11827, - "26": 1.28153, - "27": 0.27623, - "28": 0.4275, - "29": 1.02066, - "30": 0.24745, - "31": 0.19409, - "32": 0.85703, - "33": 0.71863, - "34": 0.72464, - "35": 0.68259, - "36": 0.53577, - "37": 0.69604, - "38": 0.48495, - "39": 0.68369, - "40": 0.73251, - "41": 1.04964, - "42": 0.39034, - "43": 0.29345, - "44": 1.01337, - "45": 0.51695, - "46": 0.58453, - "47": 0.83784, - "48": 0.59328, - "49": 0.27611, - "50": 0.93425, - "51": 0.15286, - "52": 0.15528, - "53": 0.78202, - "54": 0.79178, - "55": 0.64843, - "56": 0.39849, - "57": 0.62845, - "58": 0.5124, - "59": 0.59444, - "60": 0.61188, - "61": 0.17017, - "62": 0.87485, - "63": 0.13551, - "64": 0.94336, - "65": 0.63567, - "66": 0.67407, - "67": 0.6575, - "68": 0.22685, - "69": 0.63034, - "70": 0.87896, - "71": 0.2711, - "72": 0.54691, - "73": 0.68699, - "74": 0.41229, - "75": 0.83286, - "76": 0.38755, - "77": 0.49268, - "78": 1.4279, - "79": 0.22921, - "80": 0.43892, - "81": 0.47558, - "82": 0.75499, - "83": 0.36493, - "84": 0.59091, - "85": 0.2452, - "86": 0.49955, - "87": 1.08337, - "88": 0.30985, - "89": 0.73216, - "90": 0.38487, - "91": 0.39271, - "92": 0.70033, - "93": 0.26728, - "94": 0.54077, - "95": 0.51017, - "96": 1.31935, - "97": 0.38261, - "98": 0.43974, - "99": 0.78558, - "100": 0.46287 + "2": 4.57199, + "3": 0.13711, + "4": 0.13618, + "5": 0.13638, + "6": 0.13425, + "7": 0.13514, + "8": 0.13395, + "9": 0.13567, + "10": 0.13724, + "11": 0.13733, + "12": 0.13464, + "13": 0.13458, + "14": 0.13586, + "15": 0.13515, + "16": 0.13357, + "17": 0.14424, + "18": 0.13737, + "19": 0.13356, + "20": 0.13313, + "21": 0.1374, + "22": 0.13503, + "23": 0.13761, + "24": 0.13625, + "25": 0.13784, + "26": 0.14028, + "27": 0.13461, + "28": 0.13585, + "29": 0.13742, + "30": 0.13464, + "31": 0.1341, + "32": 0.13521, + "33": 0.14373, + "34": 0.13598, + "35": 0.13428, + "36": 0.13703, + "37": 0.13753, + "38": 0.14617, + "39": 0.13685, + "40": 0.13597, + "41": 0.13497, + "42": 0.13274, + "43": 0.13483, + "44": 0.13541, + "45": 0.13635, + "46": 0.13796, + "47": 0.13736, + "48": 0.13641, + "49": 0.1333, + "50": 0.13332, + "51": 0.1704, + "52": 0.17659, + "53": 0.15294, + "54": 0.14608, + "55": 0.14113, + "56": 0.13391, + "57": 0.13408, + "58": 0.13323, + "59": 0.13354, + "60": 0.1334, + "61": 0.13546, + "62": 0.13349, + "63": 0.13251, + "64": 0.13282, + "65": 0.13312, + "66": 0.1332, + "67": 0.1328, + "68": 0.13195, + "69": 0.13377, + "70": 0.13498, + "71": 0.13265, + "72": 0.13342, + "73": 0.13364, + "74": 0.13316, + "75": 0.13338, + "76": 0.13295, + "77": 0.13315, + "78": 0.13363, + "79": 0.13406, + "80": 0.13581, + "81": 0.13436, + "82": 0.13445, + "83": 0.13213, + "84": 0.1412, + "85": 0.13595, + "86": 0.13187, + "87": 0.1355, + "88": 0.1328, + "89": 0.13271, + "90": 0.13254, + "91": 0.13228, + "92": 0.13274, + "93": 0.13469, + "94": 0.13966, + "95": 0.13293, + "96": 0.13288, + "97": 0.13474, + "98": 0.13377, + "99": 0.13375, + "100": 0.13439 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json index 2840b9746ab..dfa85d53532 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90174, - "2": 10.90447, - "3": 10.91297, - "4": 10.90733, - "5": 10.90344, - "6": 10.88332, - "7": 10.89468, - "8": 10.90028, - "9": 10.9021, - "10": 10.90121, - "11": 10.89368, - "12": 10.88437, - "13": 10.87653, - "14": 10.87083, - "15": 10.86039, - "16": 10.85074, - "17": 10.85348, - "18": 10.83498, - "19": 10.83514, - "20": 10.75544, - "21": 10.7526, + "1": 10.90178, + "2": 10.90453, + "3": 10.91307, + "4": 10.9073, + "5": 10.90351, + "6": 10.8833, + "7": 10.89473, + "8": 10.90035, + "9": 10.90203, + "10": 10.90113, + "11": 10.8938, + "12": 10.88447, + "13": 10.87658, + "14": 10.87087, + "15": 10.86037, + "16": 10.85073, + "17": 10.85346, + "18": 10.83506, + "19": 10.83511, + "20": 10.75546, + "21": 10.75262, "22": 10.74059, - "23": 10.73225, - "24": 10.69032, - "25": 10.70418, - "26": 10.6823, - "27": 10.64975, - "28": 10.56722, + "23": 10.73224, + "24": 10.69026, + "25": 10.70424, + "26": 10.68233, + "27": 10.64986, + "28": 10.56727, "29": 10.54333, - "30": 10.51478, - "31": 10.5135, - "32": 10.49083, + "30": 10.5149, + "31": 10.51349, + "32": 10.49085, "33": 10.46441, - "34": 10.41257, - "35": 10.41932, - "36": 10.40629, - "37": 10.35435, - "38": 10.36121, - "39": 10.32238, - "40": 10.3208, - "41": 10.28486, - "42": 10.2753, - "43": 10.24448, - "44": 10.22008, + "34": 10.41259, + "35": 10.41936, + "36": 10.40637, + "37": 10.35437, + "38": 10.36126, + "39": 10.32246, + "40": 10.32086, + "41": 10.28493, + "42": 10.2754, + "43": 10.24451, + "44": 10.22011, "45": 10.23701, - "46": 10.19652, - "47": 10.184, + "46": 10.19654, + "47": 10.18403, "48": 10.13231, - "49": 10.13082, - "50": 10.14454, - "51": 10.14126, - "52": 10.09149, - "53": 10.08937, - "54": 10.06294, - "55": 10.01717, - "56": 10.07169, - "57": 10.03655, - "58": 10.06554, - "59": 10.00273, - "60": 10.01727, - "61": 9.98398, - "62": 9.937, - "63": 10.03276, - "64": 9.96992, - "65": 9.93251, - "66": 9.97456, - "67": 9.94352, - "68": 9.89683, - "69": 9.91356, - "70": 9.904, - "71": 9.93385, - "72": 9.8928, - "73": 9.87347, - "74": 9.87565, - "75": 9.84138, - "76": 9.89744, - "77": 9.88738, - "78": 9.82628, - "79": 9.83349, - "80": 9.85242, + "49": 10.13088, + "50": 10.14464, + "51": 10.14125, + "52": 10.09151, + "53": 10.08944, + "54": 10.063, + "55": 10.01719, + "56": 10.07167, + "57": 10.03664, + "58": 10.06557, + "59": 10.00279, + "60": 10.01724, + "61": 9.98395, + "62": 9.93711, + "63": 10.03279, + "64": 9.96997, + "65": 9.93255, + "66": 9.97462, + "67": 9.94357, + "68": 9.89688, + "69": 9.91361, + "70": 9.90406, + "71": 9.93394, + "72": 9.89285, + "73": 9.8735, + "74": 9.87572, + "75": 9.84139, + "76": 9.89746, + "77": 9.88736, + "78": 9.82633, + "79": 9.8335, + "80": 9.85244, "81": 9.876, - "82": 9.82812, - "83": 9.77197, - "84": 9.70915, - "85": 9.6939, - "86": 9.80488, - "87": 9.85235, - "88": 9.82811, - "89": 9.74806, - "90": 9.74109, - "91": 9.75365, - "92": 9.74941, - "93": 9.67469, + "82": 9.82819, + "83": 9.772, + "84": 9.70926, + "85": 9.694, + "86": 9.8049, + "87": 9.85239, + "88": 9.82812, + "89": 9.74807, + "90": 9.7412, + "91": 9.75372, + "92": 9.74942, + "93": 9.67471, "94": 9.75091, - "95": 9.75728, - "96": 9.73751, - "97": 9.65522, - "98": 9.69868, - "99": 9.75619, - "100": 9.63837 + "95": 9.75729, + "96": 9.73755, + "97": 9.65533, + "98": 9.69873, + "99": 9.75617, + "100": 9.6384 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1978.0, - "2": 1895.0, - "3": 1998.0, - "4": 1966.0, - "5": 1875.0, - "6": 1861.0, - "7": 2170.0, - "8": 1843.0, - "9": 1944.0, - "10": 1981.0, - "11": 1941.0, - "12": 1919.0, - "13": 1998.0, - "14": 2075.0, - "15": 1740.0, - "16": 1985.0, - "17": 2016.0, - "18": 2009.0, - "19": 1899.0, - "20": 1950.0, - "21": 2023.0, - "22": 2029.0, - "23": 1864.0, - "24": 2101.0, - "25": 1890.0, - "26": 1878.0, - "27": 2006.0, - "28": 1946.0, - "29": 2100.0, - "30": 1930.0, - "31": 1962.0, - "32": 1987.0, - "33": 2055.0, - "34": 2071.0, - "35": 2105.0, - "36": 2062.0, - "37": 2260.0, - "38": 2200.0, - "39": 2203.0, - "40": 2429.0, - "41": 2313.0, - "42": 2009.0, - "43": 2315.0, - "44": 2252.0, - "45": 2617.0, - "46": 2370.0, - "47": 2401.0, - "48": 2580.0, - "49": 2660.0, - "50": 2554.0, - "51": 2637.0, - "52": 2630.0, - "53": 2566.0, - "54": 2752.0, - "55": 2425.0, - "56": 2821.0, - "57": 2290.0, - "58": 3193.0, - "59": 2711.0, - "60": 2823.0, - "61": 2724.0, - "62": 2944.0, - "63": 3036.0, - "64": 3205.0, - "65": 2564.0, - "66": 2898.0, - "67": 3355.0, - "68": 3084.0, - "69": 3068.0, - "70": 2941.0, - "71": 2941.0, - "72": 3184.0, - "73": 3234.0, - "74": 3085.0, - "75": 3120.0, - "76": 3286.0, - "77": 3099.0, - "78": 3156.0, - "79": 2924.0, - "80": 3054.0, - "81": 3184.0, - "82": 3181.0, - "83": 2900.0, - "84": 2870.0, - "85": 2927.0, - "86": 3155.0, - "87": 3329.0, - "88": 3293.0, - "89": 3022.0, - "90": 3468.0, - "91": 2884.0, - "92": 2866.0, - "93": 3052.0, - "94": 3389.0, - "95": 3208.0, - "96": 3251.0, - "97": 3320.0, - "98": 3483.0, - "99": 3022.0, - "100": 3058.0 + "1": 2002.0, + "2": 1836.0, + "3": 1999.0, + "4": 2016.0, + "5": 1979.0, + "6": 1886.0, + "7": 2156.0, + "8": 1876.0, + "9": 1888.0, + "10": 2089.0, + "11": 1977.0, + "12": 1914.0, + "13": 2062.0, + "14": 2158.0, + "15": 1776.0, + "16": 2023.0, + "17": 2071.0, + "18": 2037.0, + "19": 1910.0, + "20": 1877.0, + "21": 2070.0, + "22": 2000.0, + "23": 1827.0, + "24": 1955.0, + "25": 1929.0, + "26": 1983.0, + "27": 2025.0, + "28": 1908.0, + "29": 2104.0, + "30": 1924.0, + "31": 1992.0, + "32": 2010.0, + "33": 1999.0, + "34": 2167.0, + "35": 2124.0, + "36": 2153.0, + "37": 2278.0, + "38": 2169.0, + "39": 2249.0, + "40": 2403.0, + "41": 2380.0, + "42": 2181.0, + "43": 2373.0, + "44": 2225.0, + "45": 2569.0, + "46": 2340.0, + "47": 2421.0, + "48": 2506.0, + "49": 2700.0, + "50": 2597.0, + "51": 2587.0, + "52": 2682.0, + "53": 2576.0, + "54": 2762.0, + "55": 2463.0, + "56": 2721.0, + "57": 2338.0, + "58": 3163.0, + "59": 2810.0, + "60": 2899.0, + "61": 2673.0, + "62": 2957.0, + "63": 3113.0, + "64": 3081.0, + "65": 2515.0, + "66": 2833.0, + "67": 3401.0, + "68": 3042.0, + "69": 3007.0, + "70": 3051.0, + "71": 2909.0, + "72": 3091.0, + "73": 3306.0, + "74": 3079.0, + "75": 3091.0, + "76": 3238.0, + "77": 3202.0, + "78": 3225.0, + "79": 2914.0, + "80": 2992.0, + "81": 3244.0, + "82": 3245.0, + "83": 2860.0, + "84": 3002.0, + "85": 2944.0, + "86": 3275.0, + "87": 3306.0, + "88": 3234.0, + "89": 3091.0, + "90": 3319.0, + "91": 2865.0, + "92": 2913.0, + "93": 3038.0, + "94": 3317.0, + "95": 3212.0, + "96": 3275.0, + "97": 3311.0, + "98": 3360.0, + "99": 3145.0, + "100": 2942.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.85505, - "3": 1.19024, - "4": 0.84359, - "5": 0.76463, - "6": 0.87917, - "7": 0.66058, - "8": 1.20151, - "9": 0.79788, - "10": 1.24536, - "11": 0.82619, - "12": 1.1669, - "13": 0.36186, - "14": 1.36446, - "15": 0.81937, - "16": 0.67434, - "17": 1.2472, - "18": 0.88548, - "19": 1.01634, - "20": 0.9387, - "21": 0.95791, - "22": 0.91143, - "23": 0.85542, - "24": 1.45269, - "25": 0.65296, - "26": 1.10984, - "27": 0.61392, - "28": 0.83337, - "29": 1.07639, - "30": 0.87011, - "31": 0.83554, - "32": 1.61682, - "33": 0.96985, - "34": 0.23125, - "35": 1.26054, - "36": 1.13629, - "37": 0.83452, - "38": 0.53082, - "39": 1.11374, - "40": 1.37277, - "41": 0.91611, - "42": 0.94958, - "43": 0.65567, - "44": 1.24464, - "45": 0.84205, - "46": 0.82381, - "47": 1.10749, - "48": 0.82344, - "49": 0.72087, - "50": 1.07304, - "51": 0.72789, - "52": 1.08556, - "53": 0.90301, - "54": 0.94685, - "55": 1.05426, - "56": 1.08655, - "57": 0.76066, - "58": 1.26044, - "59": 0.8244, - "60": 0.58204, - "61": 1.11386, - "62": 0.94042, - "63": 1.28285, - "64": 1.15642, - "65": 0.65714, - "66": 0.98269, - "67": 0.97012, - "68": 0.818, - "69": 1.08371, - "70": 0.77491, - "71": 1.17647, - "72": 1.18433, - "73": 0.69762, - "74": 0.98057, - "75": 0.97667, - "76": 0.83648, - "77": 0.74425, - "78": 0.85482, - "79": 1.36084, - "80": 1.01719, - "81": 1.01442, - "82": 1.03397, - "83": 0.95018, - "84": 0.73442, - "85": 0.74985, - "86": 1.22632, - "87": 0.81708, - "88": 1.11293, - "89": 1.18558, - "90": 1.05032, - "91": 0.93457, - "92": 1.52191, - "93": 1.15913, - "94": 1.25951, - "95": 1.06046, - "96": 0.86132, - "97": 0.84724, - "98": 1.10731, - "99": 1.51862, - "100": 0.84245 + "2": 3.90523, + "3": 1.29223, + "4": 0.86685, + "5": 0.7473, + "6": 1.03999, + "7": 1.13343, + "8": 1.3715, + "9": 1.65339, + "10": 0.60932, + "11": 0.99294, + "12": 0.99111, + "13": 1.0168, + "14": 1.15016, + "15": 1.04523, + "16": 0.91249, + "17": 1.34205, + "18": 0.9984, + "19": 0.95686, + "20": 1.08248, + "21": 1.07765, + "22": 0.97475, + "23": 0.93956, + "24": 1.56487, + "25": 0.44866, + "26": 0.85624, + "27": 0.88043, + "28": 1.47918, + "29": 0.58599, + "30": 0.79304, + "31": 1.00307, + "32": 1.14514, + "33": 0.98392, + "34": 0.97371, + "35": 1.30043, + "36": 0.74123, + "37": 0.79848, + "38": 0.70306, + "39": 1.16397, + "40": 1.31245, + "41": 1.21852, + "42": 1.24782, + "43": 0.72573, + "44": 1.36276, + "45": 1.28662, + "46": 0.8781, + "47": 1.11129, + "48": 0.89197, + "49": 0.80923, + "50": 0.9543, + "51": 0.70996, + "52": 1.00472, + "53": 0.97277, + "54": 0.81506, + "55": 1.17217, + "56": 0.78525, + "57": 0.72956, + "58": 1.26042, + "59": 1.05084, + "60": 0.5839, + "61": 0.90734, + "62": 1.23282, + "63": 1.00984, + "64": 1.11717, + "65": 0.70943, + "66": 1.16805, + "67": 1.2144, + "68": 0.84332, + "69": 1.30064, + "70": 0.85054, + "71": 1.10419, + "72": 0.94316, + "73": 0.98618, + "74": 0.49709, + "75": 0.56365, + "76": 1.12418, + "77": 1.30192, + "78": 1.19181, + "79": 1.63377, + "80": 1.09257, + "81": 0.9474, + "82": 1.4304, + "83": 1.18743, + "84": 1.02619, + "85": 0.68573, + "86": 0.9524, + "87": 1.39549, + "88": 0.76122, + "89": 0.82357, + "90": 1.99041, + "91": 0.75711, + "92": 0.85945, + "93": 0.78569, + "94": 1.45739, + "95": 0.91367, + "96": 0.76272, + "97": 0.84334, + "98": 0.5072, + "99": 1.06603, + "100": 0.92562 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json index 02f836290c2..ff09d631b6f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95939, - "2": 10.94561, - "3": 10.94918, + "1": 10.95945, + "2": 10.94558, + "3": 10.94922, "4": 10.94704, - "5": 10.9559, - "6": 10.95226, - "7": 10.95018, - "8": 10.95679, - "9": 10.94257, - "10": 10.94725, - "11": 10.9355, + "5": 10.95589, + "6": 10.95218, + "7": 10.95016, + "8": 10.95689, + "9": 10.94261, + "10": 10.94724, + "11": 10.93546, "12": 10.94741, - "13": 10.9199, - "14": 10.92082, - "15": 10.90466, - "16": 10.89239, - "17": 10.89288, - "18": 10.89246, - "19": 10.87971, - "20": 10.80737, - "21": 10.78841, - "22": 10.77824, - "23": 10.78255, - "24": 10.75943, - "25": 10.75771, - "26": 10.73464, - "27": 10.69864, - "28": 10.61058, - "29": 10.57945, - "30": 10.56536, - "31": 10.56477, - "32": 10.55151, - "33": 10.52212, - "34": 10.47131, - "35": 10.48348, - "36": 10.46229, - "37": 10.42487, - "38": 10.42072, - "39": 10.39099, - "40": 10.37297, - "41": 10.35274, - "42": 10.35024, - "43": 10.31689, - "44": 10.30513, - "45": 10.2969, - "46": 10.26609, - "47": 10.25516, - "48": 10.21044, - "49": 10.20228, - "50": 10.21713, - "51": 10.21029, - "52": 10.15829, - "53": 10.15994, - "54": 10.13462, - "55": 10.11077, - "56": 10.12861, - "57": 10.11008, - "58": 10.12175, - "59": 10.06878, - "60": 10.08995, - "61": 10.04525, - "62": 10.01558, - "63": 10.08025, - "64": 10.03236, - "65": 10.00171, - "66": 10.04585, - "67": 10.01598, - "68": 9.98431, - "69": 9.99678, - "70": 9.98004, - "71": 10.00125, - "72": 9.98844, + "13": 10.91988, + "14": 10.92077, + "15": 10.90467, + "16": 10.8924, + "17": 10.89279, + "18": 10.89248, + "19": 10.87973, + "20": 10.80736, + "21": 10.78846, + "22": 10.77833, + "23": 10.78262, + "24": 10.75945, + "25": 10.75778, + "26": 10.73468, + "27": 10.69867, + "28": 10.61066, + "29": 10.57956, + "30": 10.56539, + "31": 10.56486, + "32": 10.55161, + "33": 10.52219, + "34": 10.47136, + "35": 10.48355, + "36": 10.46232, + "37": 10.42496, + "38": 10.42078, + "39": 10.39103, + "40": 10.37295, + "41": 10.35282, + "42": 10.35034, + "43": 10.31696, + "44": 10.30514, + "45": 10.29693, + "46": 10.26616, + "47": 10.2552, + "48": 10.21043, + "49": 10.2023, + "50": 10.21715, + "51": 10.21037, + "52": 10.1583, + "53": 10.15998, + "54": 10.13463, + "55": 10.11084, + "56": 10.12869, + "57": 10.11006, + "58": 10.12183, + "59": 10.06883, + "60": 10.09, + "61": 10.0453, + "62": 10.01564, + "63": 10.08031, + "64": 10.03238, + "65": 10.00175, + "66": 10.04593, + "67": 10.01595, + "68": 9.98437, + "69": 9.99683, + "70": 9.98003, + "71": 10.00129, + "72": 9.98841, "73": 9.97265, - "74": 9.96397, - "75": 9.92908, - "76": 9.95782, - "77": 9.95865, - "78": 9.91024, - "79": 9.9085, - "80": 9.92434, - "81": 9.94839, - "82": 9.8887, - "83": 9.85304, - "84": 9.79711, - "85": 9.7777, - "86": 9.88374, - "87": 9.90613, - "88": 9.87919, - "89": 9.81825, - "90": 9.81548, - "91": 9.81755, - "92": 9.81598, - "93": 9.74136, - "94": 9.82061, - "95": 9.80831, - "96": 9.79427, - "97": 9.73959, - "98": 9.76307, - "99": 9.81733, - "100": 9.70258 + "74": 9.96398, + "75": 9.92912, + "76": 9.95785, + "77": 9.95872, + "78": 9.91029, + "79": 9.90855, + "80": 9.92441, + "81": 9.94842, + "82": 9.88868, + "83": 9.85308, + "84": 9.79712, + "85": 9.77773, + "86": 9.88383, + "87": 9.90616, + "88": 9.87921, + "89": 9.8183, + "90": 9.81557, + "91": 9.81758, + "92": 9.81601, + "93": 9.74139, + "94": 9.82066, + "95": 9.80836, + "96": 9.79432, + "97": 9.73962, + "98": 9.76311, + "99": 9.81735, + "100": 9.70263 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1688.0, - "2": 1724.0, - "3": 1783.0, - "4": 1601.0, - "5": 1744.0, - "6": 1688.0, - "7": 1868.0, - "8": 1702.0, - "9": 1745.0, - "10": 1683.0, - "11": 1563.0, - "12": 1636.0, - "13": 1778.0, - "14": 1773.0, - "15": 1694.0, + "1": 1757.0, + "2": 1634.0, + "3": 1778.0, + "4": 1689.0, + "5": 1718.0, + "6": 1665.0, + "7": 1932.0, + "8": 1688.0, + "9": 1691.0, + "10": 1675.0, + "11": 1611.0, + "12": 1641.0, + "13": 1752.0, + "14": 1772.0, + "15": 1704.0, "16": 1707.0, - "17": 1779.0, - "18": 1679.0, - "19": 1620.0, - "20": 1646.0, - "21": 1760.0, - "22": 1708.0, - "23": 1621.0, - "24": 1776.0, - "25": 1673.0, - "26": 1769.0, - "27": 1866.0, - "28": 1756.0, - "29": 1903.0, - "30": 1877.0, - "31": 2055.0, - "32": 2002.0, - "33": 2047.0, - "34": 1991.0, - "35": 2000.0, - "36": 1885.0, - "37": 2216.0, - "38": 2158.0, - "39": 2212.0, - "40": 2320.0, - "41": 2350.0, - "42": 2074.0, - "43": 2261.0, - "44": 2201.0, - "45": 2469.0, - "46": 2322.0, - "47": 2456.0, - "48": 2498.0, - "49": 2759.0, - "50": 2514.0, - "51": 2446.0, - "52": 2598.0, - "53": 2592.0, - "54": 2706.0, - "55": 2471.0, - "56": 2631.0, - "57": 2252.0, - "58": 3577.0, - "59": 2890.0, - "60": 2988.0, - "61": 2621.0, - "62": 3002.0, - "63": 3226.0, - "64": 3493.0, - "65": 2609.0, - "66": 2957.0, - "67": 3658.0, - "68": 3215.0, - "69": 2894.0, - "70": 3265.0, - "71": 3065.0, - "72": 2937.0, - "73": 3491.0, - "74": 3258.0, - "75": 2911.0, - "76": 3278.0, - "77": 3785.0, - "78": 3230.0, - "79": 3147.0, - "80": 3018.0, - "81": 3468.0, - "82": 3034.0, - "83": 3111.0, - "84": 2921.0, - "85": 2608.0, - "86": 2990.0, - "87": 2705.0, - "88": 3045.0, - "89": 3038.0, - "90": 3834.0, - "91": 3046.0, - "92": 3063.0, - "93": 2974.0, - "94": 3253.0, - "95": 3215.0, - "96": 3509.0, - "97": 3520.0, - "98": 3331.0, - "99": 3062.0, - "100": 2906.0 + "17": 1774.0, + "18": 1653.0, + "19": 1654.0, + "20": 1674.0, + "21": 1884.0, + "22": 1717.0, + "23": 1682.0, + "24": 1842.0, + "25": 1672.0, + "26": 1827.0, + "27": 1781.0, + "28": 1787.0, + "29": 1874.0, + "30": 1756.0, + "31": 2047.0, + "32": 2000.0, + "33": 2016.0, + "34": 1927.0, + "35": 2059.0, + "36": 1953.0, + "37": 2219.0, + "38": 2079.0, + "39": 2287.0, + "40": 2267.0, + "41": 2353.0, + "42": 1984.0, + "43": 2366.0, + "44": 2237.0, + "45": 2406.0, + "46": 2437.0, + "47": 2490.0, + "48": 2610.0, + "49": 2824.0, + "50": 2462.0, + "51": 2550.0, + "52": 2650.0, + "53": 2604.0, + "54": 2782.0, + "55": 2379.0, + "56": 2607.0, + "57": 2254.0, + "58": 3543.0, + "59": 2766.0, + "60": 2903.0, + "61": 2703.0, + "62": 3189.0, + "63": 3372.0, + "64": 3554.0, + "65": 2665.0, + "66": 2967.0, + "67": 3732.0, + "68": 3289.0, + "69": 2882.0, + "70": 3230.0, + "71": 3124.0, + "72": 2888.0, + "73": 3497.0, + "74": 3070.0, + "75": 2934.0, + "76": 3294.0, + "77": 3651.0, + "78": 3172.0, + "79": 3231.0, + "80": 3072.0, + "81": 3623.0, + "82": 3038.0, + "83": 3112.0, + "84": 2955.0, + "85": 2647.0, + "86": 2960.0, + "87": 2632.0, + "88": 2986.0, + "89": 3002.0, + "90": 3706.0, + "91": 2999.0, + "92": 2941.0, + "93": 2999.0, + "94": 3127.0, + "95": 3168.0, + "96": 3607.0, + "97": 3401.0, + "98": 3405.0, + "99": 2933.0, + "100": 3025.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 746194432.0, - "2": 746194432.0, - "3": 746194432.0, - "4": 746194432.0, - "5": 746194432.0, - "6": 746194432.0, - "7": 746194432.0, - "8": 746194432.0, - "9": 746194432.0, - "10": 746194432.0, - "11": 746194432.0, - "12": 746194432.0, - "13": 746194432.0, - "14": 746194432.0, - "15": 746194432.0, - "16": 746194432.0, - "17": 746194432.0, - "18": 746194432.0, - "19": 746194432.0, - "20": 746194432.0, - "21": 746194432.0, - "22": 746194432.0, - "23": 746194432.0, - "24": 746194432.0, - "25": 746194432.0, - "26": 746194432.0, - "27": 746194432.0, - "28": 746194432.0, - "29": 746194432.0, - "30": 746194432.0, - "31": 746194432.0, - "32": 746194432.0, - "33": 746194432.0, - "34": 746194432.0, - "35": 746194432.0, - "36": 746194432.0, - "37": 746194432.0, - "38": 746194432.0, - "39": 746194432.0, - "40": 746194432.0, - "41": 746194432.0, - "42": 746194432.0, - "43": 746194432.0, - "44": 746194432.0, - "45": 746194432.0, - "46": 746194432.0, - "47": 746194432.0, - "48": 746194432.0, - "49": 746194432.0, - "50": 746194432.0, - "51": 746194432.0, - "52": 746194432.0, - "53": 746194432.0, - "54": 746194432.0, - "55": 746194432.0, - "56": 746194432.0, - "57": 746194432.0, - "58": 746194432.0, - "59": 746194432.0, - "60": 746194432.0, - "61": 746194432.0, - "62": 746194432.0, - "63": 746194432.0, - "64": 746194432.0, - "65": 746194432.0, - "66": 746194432.0, - "67": 746194432.0, - "68": 746194432.0, - "69": 746194432.0, - "70": 746194432.0, - "71": 746194432.0, - "72": 746194432.0, - "73": 746194432.0, - "74": 746194432.0, - "75": 746194432.0, - "76": 746194432.0, - "77": 746194432.0, - "78": 746194432.0, - "79": 746194432.0, - "80": 746194432.0, - "81": 746194432.0, - "82": 746194432.0, - "83": 746194432.0, - "84": 746194432.0, - "85": 746194432.0, - "86": 746194432.0, - "87": 746194432.0, - "88": 746194432.0, - "89": 746194432.0, - "90": 746194432.0, - "91": 746194432.0, - "92": 746194432.0, - "93": 746194432.0, - "94": 746194432.0, - "95": 746194432.0, - "96": 746194432.0, - "97": 746194432.0, - "98": 746194432.0, - "99": 746194432.0, - "100": 746194432.0 + "1": 747244032.0, + "2": 747244032.0, + "3": 747244032.0, + "4": 747244032.0, + "5": 747244032.0, + "6": 747244032.0, + "7": 747244032.0, + "8": 747244032.0, + "9": 747244032.0, + "10": 747244032.0, + "11": 747244032.0, + "12": 747244032.0, + "13": 747244032.0, + "14": 747244032.0, + "15": 747244032.0, + "16": 747244032.0, + "17": 747244032.0, + "18": 747244032.0, + "19": 747244032.0, + "20": 747244032.0, + "21": 747244032.0, + "22": 747244032.0, + "23": 747244032.0, + "24": 747244032.0, + "25": 747244032.0, + "26": 747244032.0, + "27": 747244032.0, + "28": 747244032.0, + "29": 747244032.0, + "30": 747244032.0, + "31": 747244032.0, + "32": 747244032.0, + "33": 747244032.0, + "34": 747244032.0, + "35": 747244032.0, + "36": 747244032.0, + "37": 747244032.0, + "38": 747244032.0, + "39": 747244032.0, + "40": 747244032.0, + "41": 747244032.0, + "42": 747244032.0, + "43": 747244032.0, + "44": 747244032.0, + "45": 747244032.0, + "46": 747244032.0, + "47": 747244032.0, + "48": 747244032.0, + "49": 747244032.0, + "50": 747244032.0, + "51": 747244032.0, + "52": 747244032.0, + "53": 747244032.0, + "54": 747244032.0, + "55": 747244032.0, + "56": 747244032.0, + "57": 747244032.0, + "58": 747244032.0, + "59": 747244032.0, + "60": 747244032.0, + "61": 747244032.0, + "62": 747244032.0, + "63": 747244032.0, + "64": 747244032.0, + "65": 747244032.0, + "66": 747244032.0, + "67": 747244032.0, + "68": 747244032.0, + "69": 747244032.0, + "70": 747244032.0, + "71": 747244032.0, + "72": 747244032.0, + "73": 747244032.0, + "74": 747244032.0, + "75": 747244032.0, + "76": 747244032.0, + "77": 747244032.0, + "78": 747244032.0, + "79": 747244032.0, + "80": 747244032.0, + "81": 747244032.0, + "82": 747244032.0, + "83": 747244032.0, + "84": 747244032.0, + "85": 747244032.0, + "86": 747244032.0, + "87": 747244032.0, + "88": 747244032.0, + "89": 747244032.0, + "90": 747244032.0, + "91": 747244032.0, + "92": 747244032.0, + "93": 747244032.0, + "94": 747244032.0, + "95": 747244032.0, + "96": 747244032.0, + "97": 747244032.0, + "98": 747244032.0, + "99": 747244032.0, + "100": 747244032.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1927202816.0, - "2": 2209851392.0, - "3": 2209851392.0, - "4": 2209851392.0, - "5": 2209851392.0, - "6": 2209851392.0, - "7": 2209851392.0, - "8": 2209851392.0, - "9": 2209851392.0, - "10": 2209851392.0, - "11": 2209851392.0, - "12": 2209851392.0, - "13": 2209851392.0, - "14": 2209851392.0, - "15": 2209851392.0, - "16": 2209851392.0, - "17": 2209851392.0, - "18": 2209851392.0, - "19": 2209851392.0, - "20": 2209851392.0, - "21": 2209851392.0, - "22": 2209851392.0, - "23": 2209851392.0, - "24": 2209851392.0, - "25": 2209851392.0, - "26": 2209851392.0, - "27": 2209851392.0, - "28": 2209851392.0, - "29": 2209851392.0, - "30": 2209851392.0, - "31": 2209851392.0, - "32": 2209851392.0, - "33": 2209851392.0, - "34": 2209851392.0, - "35": 2209851392.0, - "36": 2209851392.0, - "37": 2209851392.0, - "38": 2209851392.0, - "39": 2209851392.0, - "40": 2209851392.0, - "41": 2209851392.0, - "42": 2209851392.0, - "43": 2209851392.0, - "44": 2209851392.0, - "45": 2209851392.0, - "46": 2209851392.0, - "47": 2209851392.0, - "48": 2209851392.0, - "49": 2209851392.0, - "50": 2209851392.0, - "51": 2209851392.0, - "52": 2209851392.0, - "53": 2209851392.0, - "54": 2209851392.0, - "55": 2209851392.0, - "56": 2209851392.0, - "57": 2209851392.0, - "58": 2209851392.0, - "59": 2209851392.0, - "60": 2209851392.0, - "61": 2209851392.0, - "62": 2209851392.0, - "63": 2209851392.0, - "64": 2209851392.0, - "65": 2209851392.0, - "66": 2209851392.0, - "67": 2209851392.0, - "68": 2209851392.0, - "69": 2209851392.0, - "70": 2209851392.0, - "71": 2209851392.0, - "72": 2209851392.0, - "73": 2209851392.0, - "74": 2209851392.0, - "75": 2209851392.0, - "76": 2209851392.0, - "77": 2209851392.0, - "78": 2209851392.0, - "79": 2209851392.0, - "80": 2209851392.0, - "81": 2209851392.0, - "82": 2209851392.0, - "83": 2209851392.0, - "84": 2209851392.0, - "85": 2209851392.0, - "86": 2209851392.0, - "87": 2209851392.0, - "88": 2209851392.0, - "89": 2209851392.0, - "90": 2209851392.0, - "91": 2209851392.0, - "92": 2209851392.0, - "93": 2209851392.0, - "94": 2209851392.0, - "95": 2209851392.0, - "96": 2209851392.0, - "97": 2209851392.0, - "98": 2209851392.0, - "99": 2209851392.0, - "100": 2209851392.0 + "2": 2211948544.0, + "3": 2211948544.0, + "4": 2211948544.0, + "5": 2211948544.0, + "6": 2211948544.0, + "7": 2211948544.0, + "8": 2211948544.0, + "9": 2211948544.0, + "10": 2211948544.0, + "11": 2211948544.0, + "12": 2211948544.0, + "13": 2211948544.0, + "14": 2211948544.0, + "15": 2211948544.0, + "16": 2211948544.0, + "17": 2211948544.0, + "18": 2211948544.0, + "19": 2211948544.0, + "20": 2211948544.0, + "21": 2211948544.0, + "22": 2211948544.0, + "23": 2211948544.0, + "24": 2211948544.0, + "25": 2211948544.0, + "26": 2211948544.0, + "27": 2211948544.0, + "28": 2211948544.0, + "29": 2211948544.0, + "30": 2211948544.0, + "31": 2211948544.0, + "32": 2211948544.0, + "33": 2211948544.0, + "34": 2211948544.0, + "35": 2211948544.0, + "36": 2211948544.0, + "37": 2211948544.0, + "38": 2211948544.0, + "39": 2211948544.0, + "40": 2211948544.0, + "41": 2211948544.0, + "42": 2211948544.0, + "43": 2211948544.0, + "44": 2211948544.0, + "45": 2211948544.0, + "46": 2211948544.0, + "47": 2211948544.0, + "48": 2211948544.0, + "49": 2211948544.0, + "50": 2211948544.0, + "51": 2211948544.0, + "52": 2211948544.0, + "53": 2211948544.0, + "54": 2211948544.0, + "55": 2211948544.0, + "56": 2211948544.0, + "57": 2211948544.0, + "58": 2211948544.0, + "59": 2211948544.0, + "60": 2211948544.0, + "61": 2211948544.0, + "62": 2211948544.0, + "63": 2211948544.0, + "64": 2211948544.0, + "65": 2211948544.0, + "66": 2211948544.0, + "67": 2211948544.0, + "68": 2211948544.0, + "69": 2211948544.0, + "70": 2211948544.0, + "71": 2211948544.0, + "72": 2211948544.0, + "73": 2211948544.0, + "74": 2211948544.0, + "75": 2211948544.0, + "76": 2211948544.0, + "77": 2211948544.0, + "78": 2211948544.0, + "79": 2211948544.0, + "80": 2211948544.0, + "81": 2211948544.0, + "82": 2211948544.0, + "83": 2211948544.0, + "84": 2211948544.0, + "85": 2211948544.0, + "86": 2211948544.0, + "87": 2211948544.0, + "88": 2211948544.0, + "89": 2211948544.0, + "90": 2211948544.0, + "91": 2211948544.0, + "92": 2211948544.0, + "93": 2211948544.0, + "94": 2211948544.0, + "95": 2211948544.0, + "96": 2211948544.0, + "97": 2211948544.0, + "98": 2211948544.0, + "99": 2211948544.0, + "100": 2211948544.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21329, - "3": 0.12134, - "4": 0.12225, - "5": 0.11934, - "6": 0.11821, - "7": 0.11851, - "8": 0.1188, - "9": 0.11853, - "10": 0.1179, - "11": 0.11867, - "12": 0.12117, - "13": 0.11852, - "14": 0.12764, - "15": 0.11801, - "16": 0.11893, - "17": 0.11882, - "18": 0.11769, - "19": 0.12406, - "20": 0.11743, - "21": 0.11967, - "22": 0.1188, - "23": 0.11839, + "2": 7.47187, + "3": 0.12583, + "4": 0.12513, + "5": 0.12312, + "6": 0.12002, + "7": 0.12266, + "8": 0.12083, + "9": 0.12109, + "10": 0.12778, + "11": 0.12, + "12": 0.11977, + "13": 0.1209, + "14": 0.1222, + "15": 0.11966, + "16": 0.1203, + "17": 0.12112, + "18": 0.12089, + "19": 0.11949, + "20": 0.12319, + "21": 0.11956, + "22": 0.12175, + "23": 0.11982, "24": 0.11915, - "25": 0.121, - "26": 0.1195, - "27": 0.1205, - "28": 0.11935, - "29": 0.11885, - "30": 0.11851, - "31": 0.12896, - "32": 0.11987, - "33": 0.11824, - "34": 0.11848, - "35": 0.11849, - "36": 0.11929, - "37": 0.11878, - "38": 0.1183, - "39": 0.12395, - "40": 0.1189, - "41": 0.1193, - "42": 0.12262, - "43": 0.12068, - "44": 0.12011, - "45": 0.12003, - "46": 0.11995, - "47": 0.11961, - "48": 0.11858, - "49": 0.11919, - "50": 0.12178, - "51": 0.16041, - "52": 0.15706, - "53": 0.14154, - "54": 0.12546, - "55": 0.11932, - "56": 0.11823, - "57": 0.11686, - "58": 0.11871, - "59": 0.11852, - "60": 0.11691, - "61": 0.11768, - "62": 0.11687, - "63": 0.11776, - "64": 0.11717, - "65": 0.11979, - "66": 0.11765, - "67": 0.11776, - "68": 0.11694, - "69": 0.11643, - "70": 0.11696, - "71": 0.11721, - "72": 0.11987, - "73": 0.12135, - "74": 0.11791, - "75": 0.11866, - "76": 0.11895, - "77": 0.11941, - "78": 0.12199, - "79": 0.11977, - "80": 0.12163, - "81": 0.1193, - "82": 0.1188, - "83": 0.11977, - "84": 0.11865, - "85": 0.11971, - "86": 0.11697, - "87": 0.1178, - "88": 0.11886, - "89": 0.11687, - "90": 0.11849, - "91": 0.11798, - "92": 0.11687, - "93": 0.11878, - "94": 0.11831, - "95": 0.11793, - "96": 0.11984, - "97": 0.11979, - "98": 0.11871, - "99": 0.11966, - "100": 0.11783 + "25": 0.11883, + "26": 0.1211, + "27": 0.11897, + "28": 0.11868, + "29": 0.1187, + "30": 0.1185, + "31": 0.11891, + "32": 0.12011, + "33": 0.11819, + "34": 0.11956, + "35": 0.11916, + "36": 0.12349, + "37": 0.12261, + "38": 0.12218, + "39": 0.12343, + "40": 0.12294, + "41": 0.12401, + "42": 0.13039, + "43": 0.12135, + "44": 0.12159, + "45": 0.12059, + "46": 0.12055, + "47": 0.12061, + "48": 0.12205, + "49": 0.12274, + "50": 0.12152, + "51": 0.15983, + "52": 0.13886, + "53": 0.12789, + "54": 0.12438, + "55": 0.12402, + "56": 0.12066, + "57": 0.12431, + "58": 0.12428, + "59": 0.1225, + "60": 0.12216, + "61": 0.12287, + "62": 0.1233, + "63": 0.12246, + "64": 0.12156, + "65": 0.12249, + "66": 0.12311, + "67": 0.12081, + "68": 0.12265, + "69": 0.12004, + "70": 0.12046, + "71": 0.12475, + "72": 0.12229, + "73": 0.1218, + "74": 0.12074, + "75": 0.12139, + "76": 0.1207, + "77": 0.12323, + "78": 0.12329, + "79": 0.12234, + "80": 0.12214, + "81": 0.12205, + "82": 0.12476, + "83": 0.12372, + "84": 0.12074, + "85": 0.1208, + "86": 0.12117, + "87": 0.11997, + "88": 0.1188, + "89": 0.12144, + "90": 0.1191, + "91": 0.12011, + "92": 0.12118, + "93": 0.1193, + "94": 0.11984, + "95": 0.11981, + "96": 0.1197, + "97": 0.11967, + "98": 0.1204, + "99": 0.12098, + "100": 0.12041 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json index 4ecd4f0df18..4774ab7e7a4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92753, - "2": 10.92559, + "1": 10.92752, + "2": 10.92554, "3": 10.93114, - "4": 10.93741, - "5": 10.93016, - "6": 10.92355, - "7": 10.93863, - "8": 10.92774, - "9": 10.93076, - "10": 10.92802, - "11": 10.91411, - "12": 10.92298, - "13": 10.92014, - "14": 10.90614, + "4": 10.93727, + "5": 10.93015, + "6": 10.92346, + "7": 10.93864, + "8": 10.92777, + "9": 10.93074, + "10": 10.92801, + "11": 10.91409, + "12": 10.92299, + "13": 10.92006, + "14": 10.90618, "15": 10.88338, - "16": 10.87951, - "17": 10.88868, - "18": 10.86451, - "19": 10.87355, - "20": 10.783, - "21": 10.78553, - "22": 10.76764, - "23": 10.764, - "24": 10.72663, - "25": 10.73401, - "26": 10.71265, - "27": 10.67569, - "28": 10.6012, - "29": 10.58219, - "30": 10.55702, - "31": 10.56167, - "32": 10.53487, - "33": 10.50348, - "34": 10.46525, - "35": 10.47887, - "36": 10.45601, - "37": 10.41808, - "38": 10.4167, - "39": 10.38178, - "40": 10.37615, - "41": 10.34328, - "42": 10.33161, - "43": 10.3077, - "44": 10.27713, - "45": 10.2988, - "46": 10.25974, - "47": 10.23655, - "48": 10.19378, - "49": 10.18543, - "50": 10.19728, - "51": 10.19367, - "52": 10.14697, - "53": 10.14982, - "54": 10.11733, - "55": 10.08476, - "56": 10.12202, - "57": 10.10466, - "58": 10.11902, - "59": 10.06425, - "60": 10.08087, + "16": 10.87953, + "17": 10.88861, + "18": 10.86454, + "19": 10.87347, + "20": 10.78318, + "21": 10.78551, + "22": 10.76768, + "23": 10.76394, + "24": 10.72672, + "25": 10.734, + "26": 10.71269, + "27": 10.6757, + "28": 10.60128, + "29": 10.5823, + "30": 10.55707, + "31": 10.56165, + "32": 10.53484, + "33": 10.50353, + "34": 10.46529, + "35": 10.47894, + "36": 10.45605, + "37": 10.41812, + "38": 10.41677, + "39": 10.38189, + "40": 10.37623, + "41": 10.34336, + "42": 10.33171, + "43": 10.30768, + "44": 10.27724, + "45": 10.29884, + "46": 10.2598, + "47": 10.23664, + "48": 10.1938, + "49": 10.18553, + "50": 10.19738, + "51": 10.1937, + "52": 10.14694, + "53": 10.1499, + "54": 10.11736, + "55": 10.08481, + "56": 10.12206, + "57": 10.10473, + "58": 10.11908, + "59": 10.06432, + "60": 10.08092, "61": 10.03845, - "62": 10.00507, - "63": 10.07872, - "64": 10.03202, - "65": 10.00346, - "66": 10.03491, - "67": 10.00571, - "68": 9.97032, - "69": 9.99153, - "70": 9.97866, - "71": 9.9998, - "72": 9.97531, - "73": 9.96566, - "74": 9.95983, + "62": 10.00515, + "63": 10.07878, + "64": 10.03206, + "65": 10.00349, + "66": 10.03495, + "67": 10.00575, + "68": 9.97033, + "69": 9.99157, + "70": 9.97869, + "71": 9.99986, + "72": 9.97535, + "73": 9.96568, + "74": 9.95986, "75": 9.92549, - "76": 9.96566, + "76": 9.96565, "77": 9.95606, - "78": 9.90446, - "79": 9.91127, - "80": 9.92478, - "81": 9.94432, - "82": 9.88863, - "83": 9.85093, - "84": 9.78899, - "85": 9.77531, - "86": 9.88013, + "78": 9.9045, + "79": 9.91133, + "80": 9.92487, + "81": 9.94438, + "82": 9.88871, + "83": 9.85105, + "84": 9.78905, + "85": 9.77533, + "86": 9.88023, "87": 9.91309, - "88": 9.88639, - "89": 9.82366, - "90": 9.81326, - "91": 9.82136, - "92": 9.81011, - "93": 9.75216, - "94": 9.82571, - "95": 9.81912, - "96": 9.8055, - "97": 9.7436, - "98": 9.77248, - "99": 9.81661, - "100": 9.71126 + "88": 9.88646, + "89": 9.8237, + "90": 9.81331, + "91": 9.82141, + "92": 9.81015, + "93": 9.75217, + "94": 9.82575, + "95": 9.8191, + "96": 9.80552, + "97": 9.74366, + "98": 9.7725, + "99": 9.8167, + "100": 9.71129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1770.0, - "2": 1703.0, - "3": 1799.0, - "4": 1823.0, - "5": 1737.0, - "6": 1733.0, + "1": 1845.0, + "2": 1697.0, + "3": 1692.0, + "4": 1714.0, + "5": 1786.0, + "6": 1732.0, "7": 1952.0, - "8": 1738.0, - "9": 1687.0, - "10": 1780.0, - "11": 1673.0, - "12": 1677.0, - "13": 1724.0, - "14": 1837.0, - "15": 1570.0, - "16": 1727.0, - "17": 1788.0, - "18": 1780.0, - "19": 1660.0, - "20": 1694.0, - "21": 1732.0, - "22": 1727.0, - "23": 1703.0, - "24": 1799.0, - "25": 1588.0, - "26": 1817.0, - "27": 1684.0, - "28": 1866.0, - "29": 1939.0, - "30": 1970.0, - "31": 1940.0, - "32": 1908.0, - "33": 1954.0, - "34": 2074.0, - "35": 2035.0, - "36": 1969.0, - "37": 2171.0, - "38": 2085.0, - "39": 2307.0, - "40": 2354.0, - "41": 2281.0, - "42": 1999.0, - "43": 2430.0, - "44": 2201.0, - "45": 2656.0, - "46": 2281.0, - "47": 2477.0, - "48": 2552.0, - "49": 2811.0, - "50": 2570.0, - "51": 2414.0, - "52": 2753.0, - "53": 2603.0, - "54": 2894.0, - "55": 2730.0, - "56": 2607.0, - "57": 2214.0, - "58": 3601.0, - "59": 2895.0, - "60": 2853.0, - "61": 2642.0, - "62": 3082.0, - "63": 3315.0, - "64": 3551.0, - "65": 2620.0, - "66": 3064.0, - "67": 3946.0, - "68": 3280.0, - "69": 2902.0, - "70": 3285.0, - "71": 3061.0, - "72": 2994.0, - "73": 3480.0, - "74": 3147.0, - "75": 3252.0, - "76": 3151.0, - "77": 3721.0, - "78": 3131.0, - "79": 3305.0, - "80": 3197.0, - "81": 3396.0, - "82": 3230.0, - "83": 3266.0, - "84": 2830.0, - "85": 2720.0, - "86": 3098.0, - "87": 2969.0, - "88": 3044.0, - "89": 3029.0, - "90": 3586.0, - "91": 3189.0, - "92": 2969.0, - "93": 2896.0, - "94": 3171.0, - "95": 3403.0, - "96": 3597.0, - "97": 3484.0, - "98": 3353.0, - "99": 3320.0, - "100": 3368.0 + "8": 1761.0, + "9": 1829.0, + "10": 1835.0, + "11": 1692.0, + "12": 1719.0, + "13": 1771.0, + "14": 1925.0, + "15": 1565.0, + "16": 1676.0, + "17": 1839.0, + "18": 1831.0, + "19": 1657.0, + "20": 1646.0, + "21": 1758.0, + "22": 1764.0, + "23": 1679.0, + "24": 1796.0, + "25": 1803.0, + "26": 1861.0, + "27": 1680.0, + "28": 1836.0, + "29": 1974.0, + "30": 1870.0, + "31": 1991.0, + "32": 1855.0, + "33": 1929.0, + "34": 1979.0, + "35": 2026.0, + "36": 1997.0, + "37": 2154.0, + "38": 2182.0, + "39": 2162.0, + "40": 2249.0, + "41": 2357.0, + "42": 1947.0, + "43": 2334.0, + "44": 2145.0, + "45": 2601.0, + "46": 2416.0, + "47": 2528.0, + "48": 2505.0, + "49": 2807.0, + "50": 2606.0, + "51": 2529.0, + "52": 2706.0, + "53": 2546.0, + "54": 2820.0, + "55": 2723.0, + "56": 2570.0, + "57": 2215.0, + "58": 3573.0, + "59": 2904.0, + "60": 2916.0, + "61": 2765.0, + "62": 3181.0, + "63": 3220.0, + "64": 3594.0, + "65": 2595.0, + "66": 3023.0, + "67": 3840.0, + "68": 3470.0, + "69": 2927.0, + "70": 3210.0, + "71": 3080.0, + "72": 2938.0, + "73": 3455.0, + "74": 3146.0, + "75": 3283.0, + "76": 3170.0, + "77": 3808.0, + "78": 3264.0, + "79": 3266.0, + "80": 3117.0, + "81": 3423.0, + "82": 3087.0, + "83": 3214.0, + "84": 2912.0, + "85": 2742.0, + "86": 3145.0, + "87": 2981.0, + "88": 3157.0, + "89": 2922.0, + "90": 3525.0, + "91": 3112.0, + "92": 3039.0, + "93": 3001.0, + "94": 3145.0, + "95": 3363.0, + "96": 3581.0, + "97": 3603.0, + "98": 3403.0, + "99": 3370.0, + "100": 3303.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.42058, - "3": 2.9906, - "4": 2.28433, - "5": 2.31945, - "6": 2.25606, - "7": 2.65053, - "8": 2.78894, - "9": 3.17745, - "10": 2.20994, - "11": 2.7953, - "12": 2.04648, - "13": 2.62183, - "14": 2.47023, - "15": 1.89176, - "16": 2.39045, - "17": 2.96348, - "18": 2.17915, - "19": 2.73072, - "20": 2.39913, - "21": 2.50469, - "22": 2.02961, - "23": 2.85935, - "24": 2.42236, - "25": 2.32274, - "26": 2.22901, - "27": 2.10895, - "28": 2.0858, - "29": 1.81038, - "30": 2.00374, - "31": 1.79224, - "32": 2.98457, - "33": 1.94536, - "34": 1.56607, - "35": 2.24687, - "36": 2.6891, - "37": 2.1153, - "38": 1.78402, - "39": 2.49259, - "40": 1.97637, - "41": 2.38065, - "42": 2.98564, - "43": 2.14245, - "44": 2.15079, - "45": 2.79879, - "46": 2.57605, - "47": 2.80077, - "48": 3.21632, - "49": 2.19134, - "50": 2.51729, - "51": 1.92835, - "52": 2.61796, - "53": 2.33409, - "54": 2.21492, - "55": 1.80381, - "56": 2.84132, - "57": 1.31239, - "58": 2.09837, - "59": 2.19106, - "60": 1.96286, - "61": 2.73324, - "62": 2.32317, - "63": 2.02322, - "64": 2.3249, - "65": 2.3802, - "66": 2.53532, - "67": 2.51893, - "68": 1.93902, - "69": 2.55482, - "70": 2.30937, - "71": 3.43239, - "72": 2.34936, - "73": 2.13696, - "74": 2.48062, - "75": 2.74573, - "76": 2.41693, - "77": 2.97019, - "78": 2.30417, - "79": 2.38768, - "80": 2.36351, - "81": 2.57248, - "82": 2.47399, - "83": 2.40451, - "84": 1.97887, - "85": 2.05544, - "86": 2.1894, - "87": 2.6966, - "88": 2.3178, - "89": 2.48366, - "90": 2.02263, - "91": 2.45368, - "92": 2.61381, - "93": 1.61994, - "94": 2.41232, - "95": 2.31977, - "96": 2.39685, - "97": 1.61519, - "98": 2.05838, - "99": 2.51131, - "100": 1.98917 + "2": 5.58877, + "3": 2.63153, + "4": 2.55044, + "5": 1.72014, + "6": 2.30744, + "7": 1.83596, + "8": 2.7243, + "9": 2.77044, + "10": 2.22595, + "11": 2.88242, + "12": 1.77212, + "13": 1.95953, + "14": 2.20487, + "15": 1.85516, + "16": 2.04034, + "17": 2.51318, + "18": 1.93648, + "19": 2.16947, + "20": 1.72898, + "21": 2.15718, + "22": 1.8681, + "23": 1.92804, + "24": 1.37872, + "25": 2.04816, + "26": 2.09311, + "27": 1.63546, + "28": 1.63792, + "29": 1.50077, + "30": 1.93507, + "31": 1.50343, + "32": 2.537, + "33": 1.42668, + "34": 1.28672, + "35": 2.28645, + "36": 2.14402, + "37": 1.25881, + "38": 1.5499, + "39": 1.60849, + "40": 1.51634, + "41": 1.3804, + "42": 2.05335, + "43": 1.66882, + "44": 1.28412, + "45": 2.19341, + "46": 1.48578, + "47": 1.54213, + "48": 2.20251, + "49": 1.58362, + "50": 1.72887, + "51": 1.00808, + "52": 1.95501, + "53": 1.8435, + "54": 2.15731, + "55": 1.96032, + "56": 2.0755, + "57": 1.25011, + "58": 1.83177, + "59": 1.81096, + "60": 1.34493, + "61": 2.09882, + "62": 2.06309, + "63": 1.79067, + "64": 1.88028, + "65": 1.84123, + "66": 2.08685, + "67": 1.82935, + "68": 1.41402, + "69": 2.02485, + "70": 2.21294, + "71": 2.1685, + "72": 1.63812, + "73": 1.57621, + "74": 1.72901, + "75": 1.5515, + "76": 1.58363, + "77": 1.53292, + "78": 1.54417, + "79": 2.36289, + "80": 1.9647, + "81": 2.00424, + "82": 2.05961, + "83": 1.43028, + "84": 1.50598, + "85": 1.58709, + "86": 1.4184, + "87": 2.17228, + "88": 1.8678, + "89": 1.99636, + "90": 1.08673, + "91": 1.62774, + "92": 2.27084, + "93": 1.54805, + "94": 1.60309, + "95": 1.84941, + "96": 1.59494, + "97": 1.50677, + "98": 1.9873, + "99": 2.09202, + "100": 1.68921 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json index 44a40c1ca1b..ac6c28d5bea 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95939, - "2": 10.94561, - "3": 10.94918, + "1": 10.95945, + "2": 10.94558, + "3": 10.94922, "4": 10.94704, - "5": 10.9559, - "6": 10.95226, - "7": 10.95018, - "8": 10.95679, - "9": 10.94257, - "10": 10.94725, - "11": 10.9355, + "5": 10.95589, + "6": 10.95218, + "7": 10.95016, + "8": 10.95689, + "9": 10.94261, + "10": 10.94724, + "11": 10.93546, "12": 10.94741, - "13": 10.9199, - "14": 10.92082, - "15": 10.90466, - "16": 10.89239, - "17": 10.89288, - "18": 10.89246, - "19": 10.87971, - "20": 10.80737, - "21": 10.78841, - "22": 10.77824, - "23": 10.78255, - "24": 10.75943, - "25": 10.75771, - "26": 10.73464, - "27": 10.69864, - "28": 10.61058, - "29": 10.57945, - "30": 10.56536, - "31": 10.56477, - "32": 10.55151, - "33": 10.52212, - "34": 10.47131, - "35": 10.48348, - "36": 10.46229, - "37": 10.42487, - "38": 10.42072, - "39": 10.39099, - "40": 10.37297, - "41": 10.35274, - "42": 10.35024, - "43": 10.31689, - "44": 10.30513, - "45": 10.2969, - "46": 10.26609, - "47": 10.25516, - "48": 10.21044, - "49": 10.20228, - "50": 10.21713, - "51": 10.21029, - "52": 10.15829, - "53": 10.15994, - "54": 10.13462, - "55": 10.11077, - "56": 10.12861, - "57": 10.11008, - "58": 10.12175, - "59": 10.06878, - "60": 10.08995, - "61": 10.04525, - "62": 10.01558, - "63": 10.08025, - "64": 10.03236, - "65": 10.00171, - "66": 10.04585, - "67": 10.01598, - "68": 9.98431, - "69": 9.99678, - "70": 9.98004, - "71": 10.00125, - "72": 9.98844, + "13": 10.91988, + "14": 10.92077, + "15": 10.90467, + "16": 10.8924, + "17": 10.89279, + "18": 10.89248, + "19": 10.87973, + "20": 10.80736, + "21": 10.78846, + "22": 10.77833, + "23": 10.78262, + "24": 10.75945, + "25": 10.75778, + "26": 10.73468, + "27": 10.69867, + "28": 10.61066, + "29": 10.57956, + "30": 10.56539, + "31": 10.56486, + "32": 10.55161, + "33": 10.52219, + "34": 10.47136, + "35": 10.48355, + "36": 10.46232, + "37": 10.42496, + "38": 10.42078, + "39": 10.39103, + "40": 10.37295, + "41": 10.35282, + "42": 10.35034, + "43": 10.31696, + "44": 10.30514, + "45": 10.29693, + "46": 10.26616, + "47": 10.2552, + "48": 10.21043, + "49": 10.2023, + "50": 10.21715, + "51": 10.21037, + "52": 10.1583, + "53": 10.15998, + "54": 10.13463, + "55": 10.11084, + "56": 10.12869, + "57": 10.11006, + "58": 10.12183, + "59": 10.06883, + "60": 10.09, + "61": 10.0453, + "62": 10.01564, + "63": 10.08031, + "64": 10.03238, + "65": 10.00175, + "66": 10.04593, + "67": 10.01595, + "68": 9.98437, + "69": 9.99683, + "70": 9.98003, + "71": 10.00129, + "72": 9.98841, "73": 9.97265, - "74": 9.96397, - "75": 9.92908, - "76": 9.95782, - "77": 9.95865, - "78": 9.91024, - "79": 9.9085, - "80": 9.92434, - "81": 9.94839, - "82": 9.8887, - "83": 9.85304, - "84": 9.79711, - "85": 9.7777, - "86": 9.88374, - "87": 9.90613, - "88": 9.87919, - "89": 9.81825, - "90": 9.81548, - "91": 9.81755, - "92": 9.81598, - "93": 9.74136, - "94": 9.82061, - "95": 9.80831, - "96": 9.79427, - "97": 9.73959, - "98": 9.76307, - "99": 9.81733, - "100": 9.70258 + "74": 9.96398, + "75": 9.92912, + "76": 9.95785, + "77": 9.95872, + "78": 9.91029, + "79": 9.90855, + "80": 9.92441, + "81": 9.94842, + "82": 9.88868, + "83": 9.85308, + "84": 9.79712, + "85": 9.77773, + "86": 9.88383, + "87": 9.90616, + "88": 9.87921, + "89": 9.8183, + "90": 9.81557, + "91": 9.81758, + "92": 9.81601, + "93": 9.74139, + "94": 9.82066, + "95": 9.80836, + "96": 9.79432, + "97": 9.73962, + "98": 9.76311, + "99": 9.81735, + "100": 9.70263 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1688.0, - "2": 1724.0, - "3": 1783.0, - "4": 1601.0, - "5": 1744.0, - "6": 1688.0, - "7": 1868.0, - "8": 1702.0, - "9": 1745.0, - "10": 1683.0, - "11": 1563.0, - "12": 1636.0, - "13": 1778.0, - "14": 1773.0, - "15": 1694.0, + "1": 1757.0, + "2": 1634.0, + "3": 1778.0, + "4": 1689.0, + "5": 1718.0, + "6": 1665.0, + "7": 1932.0, + "8": 1688.0, + "9": 1691.0, + "10": 1675.0, + "11": 1611.0, + "12": 1641.0, + "13": 1752.0, + "14": 1772.0, + "15": 1704.0, "16": 1707.0, - "17": 1779.0, - "18": 1679.0, - "19": 1620.0, - "20": 1646.0, - "21": 1760.0, - "22": 1708.0, - "23": 1621.0, - "24": 1776.0, - "25": 1673.0, - "26": 1769.0, - "27": 1866.0, - "28": 1756.0, - "29": 1903.0, - "30": 1877.0, - "31": 2055.0, - "32": 2002.0, - "33": 2047.0, - "34": 1991.0, - "35": 2000.0, - "36": 1885.0, - "37": 2216.0, - "38": 2158.0, - "39": 2212.0, - "40": 2320.0, - "41": 2350.0, - "42": 2074.0, - "43": 2261.0, - "44": 2201.0, - "45": 2469.0, - "46": 2322.0, - "47": 2456.0, - "48": 2498.0, - "49": 2759.0, - "50": 2514.0, - "51": 2446.0, - "52": 2598.0, - "53": 2592.0, - "54": 2706.0, - "55": 2471.0, - "56": 2631.0, - "57": 2252.0, - "58": 3577.0, - "59": 2890.0, - "60": 2988.0, - "61": 2621.0, - "62": 3002.0, - "63": 3226.0, - "64": 3493.0, - "65": 2609.0, - "66": 2957.0, - "67": 3658.0, - "68": 3215.0, - "69": 2894.0, - "70": 3265.0, - "71": 3065.0, - "72": 2937.0, - "73": 3491.0, - "74": 3258.0, - "75": 2911.0, - "76": 3278.0, - "77": 3785.0, - "78": 3230.0, - "79": 3147.0, - "80": 3018.0, - "81": 3468.0, - "82": 3034.0, - "83": 3111.0, - "84": 2921.0, - "85": 2608.0, - "86": 2990.0, - "87": 2705.0, - "88": 3045.0, - "89": 3038.0, - "90": 3834.0, - "91": 3046.0, - "92": 3063.0, - "93": 2974.0, - "94": 3253.0, - "95": 3215.0, - "96": 3509.0, - "97": 3520.0, - "98": 3331.0, - "99": 3062.0, - "100": 2906.0 + "17": 1774.0, + "18": 1653.0, + "19": 1654.0, + "20": 1674.0, + "21": 1884.0, + "22": 1717.0, + "23": 1682.0, + "24": 1842.0, + "25": 1672.0, + "26": 1827.0, + "27": 1781.0, + "28": 1787.0, + "29": 1874.0, + "30": 1756.0, + "31": 2047.0, + "32": 2000.0, + "33": 2016.0, + "34": 1927.0, + "35": 2059.0, + "36": 1953.0, + "37": 2219.0, + "38": 2079.0, + "39": 2287.0, + "40": 2267.0, + "41": 2353.0, + "42": 1984.0, + "43": 2366.0, + "44": 2237.0, + "45": 2406.0, + "46": 2437.0, + "47": 2490.0, + "48": 2610.0, + "49": 2824.0, + "50": 2462.0, + "51": 2550.0, + "52": 2650.0, + "53": 2604.0, + "54": 2782.0, + "55": 2379.0, + "56": 2607.0, + "57": 2254.0, + "58": 3543.0, + "59": 2766.0, + "60": 2903.0, + "61": 2703.0, + "62": 3189.0, + "63": 3372.0, + "64": 3554.0, + "65": 2665.0, + "66": 2967.0, + "67": 3732.0, + "68": 3289.0, + "69": 2882.0, + "70": 3230.0, + "71": 3124.0, + "72": 2888.0, + "73": 3497.0, + "74": 3070.0, + "75": 2934.0, + "76": 3294.0, + "77": 3651.0, + "78": 3172.0, + "79": 3231.0, + "80": 3072.0, + "81": 3623.0, + "82": 3038.0, + "83": 3112.0, + "84": 2955.0, + "85": 2647.0, + "86": 2960.0, + "87": 2632.0, + "88": 2986.0, + "89": 3002.0, + "90": 3706.0, + "91": 2999.0, + "92": 2941.0, + "93": 2999.0, + "94": 3127.0, + "95": 3168.0, + "96": 3607.0, + "97": 3401.0, + "98": 3405.0, + "99": 2933.0, + "100": 3025.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 746194432.0, - "2": 746194432.0, - "3": 746194432.0, - "4": 746194432.0, - "5": 746194432.0, - "6": 746194432.0, - "7": 746194432.0, - "8": 746194432.0, - "9": 746194432.0, - "10": 746194432.0, - "11": 746194432.0, - "12": 746194432.0, - "13": 746194432.0, - "14": 746194432.0, - "15": 746194432.0, - "16": 746194432.0, - "17": 746194432.0, - "18": 746194432.0, - "19": 746194432.0, - "20": 746194432.0, - "21": 746194432.0, - "22": 746194432.0, - "23": 746194432.0, - "24": 746194432.0, - "25": 746194432.0, - "26": 746194432.0, - "27": 746194432.0, - "28": 746194432.0, - "29": 746194432.0, - "30": 746194432.0, - "31": 746194432.0, - "32": 746194432.0, - "33": 746194432.0, - "34": 746194432.0, - "35": 746194432.0, - "36": 746194432.0, - "37": 746194432.0, - "38": 746194432.0, - "39": 746194432.0, - "40": 746194432.0, - "41": 746194432.0, - "42": 746194432.0, - "43": 746194432.0, - "44": 746194432.0, - "45": 746194432.0, - "46": 746194432.0, - "47": 746194432.0, - "48": 746194432.0, - "49": 746194432.0, - "50": 746194432.0, - "51": 746194432.0, - "52": 746194432.0, - "53": 746194432.0, - "54": 746194432.0, - "55": 746194432.0, - "56": 746194432.0, - "57": 746194432.0, - "58": 746194432.0, - "59": 746194432.0, - "60": 746194432.0, - "61": 746194432.0, - "62": 746194432.0, - "63": 746194432.0, - "64": 746194432.0, - "65": 746194432.0, - "66": 746194432.0, - "67": 746194432.0, - "68": 746194432.0, - "69": 746194432.0, - "70": 746194432.0, - "71": 746194432.0, - "72": 746194432.0, - "73": 746194432.0, - "74": 746194432.0, - "75": 746194432.0, - "76": 746194432.0, - "77": 746194432.0, - "78": 746194432.0, - "79": 746194432.0, - "80": 746194432.0, - "81": 746194432.0, - "82": 746194432.0, - "83": 746194432.0, - "84": 746194432.0, - "85": 746194432.0, - "86": 746194432.0, - "87": 746194432.0, - "88": 746194432.0, - "89": 746194432.0, - "90": 746194432.0, - "91": 746194432.0, - "92": 746194432.0, - "93": 746194432.0, - "94": 746194432.0, - "95": 746194432.0, - "96": 746194432.0, - "97": 746194432.0, - "98": 746194432.0, - "99": 746194432.0, - "100": 746194432.0 + "1": 747244032.0, + "2": 747244032.0, + "3": 747244032.0, + "4": 747244032.0, + "5": 747244032.0, + "6": 747244032.0, + "7": 747244032.0, + "8": 747244032.0, + "9": 747244032.0, + "10": 747244032.0, + "11": 747244032.0, + "12": 747244032.0, + "13": 747244032.0, + "14": 747244032.0, + "15": 747244032.0, + "16": 747244032.0, + "17": 747244032.0, + "18": 747244032.0, + "19": 747244032.0, + "20": 747244032.0, + "21": 747244032.0, + "22": 747244032.0, + "23": 747244032.0, + "24": 747244032.0, + "25": 747244032.0, + "26": 747244032.0, + "27": 747244032.0, + "28": 747244032.0, + "29": 747244032.0, + "30": 747244032.0, + "31": 747244032.0, + "32": 747244032.0, + "33": 747244032.0, + "34": 747244032.0, + "35": 747244032.0, + "36": 747244032.0, + "37": 747244032.0, + "38": 747244032.0, + "39": 747244032.0, + "40": 747244032.0, + "41": 747244032.0, + "42": 747244032.0, + "43": 747244032.0, + "44": 747244032.0, + "45": 747244032.0, + "46": 747244032.0, + "47": 747244032.0, + "48": 747244032.0, + "49": 747244032.0, + "50": 747244032.0, + "51": 747244032.0, + "52": 747244032.0, + "53": 747244032.0, + "54": 747244032.0, + "55": 747244032.0, + "56": 747244032.0, + "57": 747244032.0, + "58": 747244032.0, + "59": 747244032.0, + "60": 747244032.0, + "61": 747244032.0, + "62": 747244032.0, + "63": 747244032.0, + "64": 747244032.0, + "65": 747244032.0, + "66": 747244032.0, + "67": 747244032.0, + "68": 747244032.0, + "69": 747244032.0, + "70": 747244032.0, + "71": 747244032.0, + "72": 747244032.0, + "73": 747244032.0, + "74": 747244032.0, + "75": 747244032.0, + "76": 747244032.0, + "77": 747244032.0, + "78": 747244032.0, + "79": 747244032.0, + "80": 747244032.0, + "81": 747244032.0, + "82": 747244032.0, + "83": 747244032.0, + "84": 747244032.0, + "85": 747244032.0, + "86": 747244032.0, + "87": 747244032.0, + "88": 747244032.0, + "89": 747244032.0, + "90": 747244032.0, + "91": 747244032.0, + "92": 747244032.0, + "93": 747244032.0, + "94": 747244032.0, + "95": 747244032.0, + "96": 747244032.0, + "97": 747244032.0, + "98": 747244032.0, + "99": 747244032.0, + "100": 747244032.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1927202816.0, - "2": 2209851392.0, - "3": 2209851392.0, - "4": 2209851392.0, - "5": 2209851392.0, - "6": 2209851392.0, - "7": 2209851392.0, - "8": 2209851392.0, - "9": 2209851392.0, - "10": 2209851392.0, - "11": 2209851392.0, - "12": 2209851392.0, - "13": 2209851392.0, - "14": 2209851392.0, - "15": 2209851392.0, - "16": 2209851392.0, - "17": 2209851392.0, - "18": 2209851392.0, - "19": 2209851392.0, - "20": 2209851392.0, - "21": 2209851392.0, - "22": 2209851392.0, - "23": 2209851392.0, - "24": 2209851392.0, - "25": 2209851392.0, - "26": 2209851392.0, - "27": 2209851392.0, - "28": 2209851392.0, - "29": 2209851392.0, - "30": 2209851392.0, - "31": 2209851392.0, - "32": 2209851392.0, - "33": 2209851392.0, - "34": 2209851392.0, - "35": 2209851392.0, - "36": 2209851392.0, - "37": 2209851392.0, - "38": 2209851392.0, - "39": 2209851392.0, - "40": 2209851392.0, - "41": 2209851392.0, - "42": 2209851392.0, - "43": 2209851392.0, - "44": 2209851392.0, - "45": 2209851392.0, - "46": 2209851392.0, - "47": 2209851392.0, - "48": 2209851392.0, - "49": 2209851392.0, - "50": 2209851392.0, - "51": 2209851392.0, - "52": 2209851392.0, - "53": 2209851392.0, - "54": 2209851392.0, - "55": 2209851392.0, - "56": 2209851392.0, - "57": 2209851392.0, - "58": 2209851392.0, - "59": 2209851392.0, - "60": 2209851392.0, - "61": 2209851392.0, - "62": 2209851392.0, - "63": 2209851392.0, - "64": 2209851392.0, - "65": 2209851392.0, - "66": 2209851392.0, - "67": 2209851392.0, - "68": 2209851392.0, - "69": 2209851392.0, - "70": 2209851392.0, - "71": 2209851392.0, - "72": 2209851392.0, - "73": 2209851392.0, - "74": 2209851392.0, - "75": 2209851392.0, - "76": 2209851392.0, - "77": 2209851392.0, - "78": 2209851392.0, - "79": 2209851392.0, - "80": 2209851392.0, - "81": 2209851392.0, - "82": 2209851392.0, - "83": 2209851392.0, - "84": 2209851392.0, - "85": 2209851392.0, - "86": 2209851392.0, - "87": 2209851392.0, - "88": 2209851392.0, - "89": 2209851392.0, - "90": 2209851392.0, - "91": 2209851392.0, - "92": 2209851392.0, - "93": 2209851392.0, - "94": 2209851392.0, - "95": 2209851392.0, - "96": 2209851392.0, - "97": 2209851392.0, - "98": 2209851392.0, - "99": 2209851392.0, - "100": 2209851392.0 + "2": 2211948544.0, + "3": 2211948544.0, + "4": 2211948544.0, + "5": 2211948544.0, + "6": 2211948544.0, + "7": 2211948544.0, + "8": 2211948544.0, + "9": 2211948544.0, + "10": 2211948544.0, + "11": 2211948544.0, + "12": 2211948544.0, + "13": 2211948544.0, + "14": 2211948544.0, + "15": 2211948544.0, + "16": 2211948544.0, + "17": 2211948544.0, + "18": 2211948544.0, + "19": 2211948544.0, + "20": 2211948544.0, + "21": 2211948544.0, + "22": 2211948544.0, + "23": 2211948544.0, + "24": 2211948544.0, + "25": 2211948544.0, + "26": 2211948544.0, + "27": 2211948544.0, + "28": 2211948544.0, + "29": 2211948544.0, + "30": 2211948544.0, + "31": 2211948544.0, + "32": 2211948544.0, + "33": 2211948544.0, + "34": 2211948544.0, + "35": 2211948544.0, + "36": 2211948544.0, + "37": 2211948544.0, + "38": 2211948544.0, + "39": 2211948544.0, + "40": 2211948544.0, + "41": 2211948544.0, + "42": 2211948544.0, + "43": 2211948544.0, + "44": 2211948544.0, + "45": 2211948544.0, + "46": 2211948544.0, + "47": 2211948544.0, + "48": 2211948544.0, + "49": 2211948544.0, + "50": 2211948544.0, + "51": 2211948544.0, + "52": 2211948544.0, + "53": 2211948544.0, + "54": 2211948544.0, + "55": 2211948544.0, + "56": 2211948544.0, + "57": 2211948544.0, + "58": 2211948544.0, + "59": 2211948544.0, + "60": 2211948544.0, + "61": 2211948544.0, + "62": 2211948544.0, + "63": 2211948544.0, + "64": 2211948544.0, + "65": 2211948544.0, + "66": 2211948544.0, + "67": 2211948544.0, + "68": 2211948544.0, + "69": 2211948544.0, + "70": 2211948544.0, + "71": 2211948544.0, + "72": 2211948544.0, + "73": 2211948544.0, + "74": 2211948544.0, + "75": 2211948544.0, + "76": 2211948544.0, + "77": 2211948544.0, + "78": 2211948544.0, + "79": 2211948544.0, + "80": 2211948544.0, + "81": 2211948544.0, + "82": 2211948544.0, + "83": 2211948544.0, + "84": 2211948544.0, + "85": 2211948544.0, + "86": 2211948544.0, + "87": 2211948544.0, + "88": 2211948544.0, + "89": 2211948544.0, + "90": 2211948544.0, + "91": 2211948544.0, + "92": 2211948544.0, + "93": 2211948544.0, + "94": 2211948544.0, + "95": 2211948544.0, + "96": 2211948544.0, + "97": 2211948544.0, + "98": 2211948544.0, + "99": 2211948544.0, + "100": 2211948544.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.23149, - "3": 0.12534, - "4": 0.12102, - "5": 0.11736, - "6": 0.12125, - "7": 0.1219, - "8": 0.11702, - "9": 0.11818, - "10": 0.1227, - "11": 0.12476, - "12": 0.12463, - "13": 0.12408, - "14": 0.12042, - "15": 0.11999, - "16": 0.11824, - "17": 0.12037, - "18": 0.11909, - "19": 0.11683, - "20": 0.1176, - "21": 0.11703, - "22": 0.11948, - "23": 0.11789, - "24": 0.12232, - "25": 0.12176, - "26": 0.11992, - "27": 0.11848, - "28": 0.12002, - "29": 0.1179, - "30": 0.11782, - "31": 0.11816, - "32": 0.11841, - "33": 0.12154, - "34": 0.12014, - "35": 0.12846, - "36": 0.12691, - "37": 0.13076, - "38": 0.13109, - "39": 0.12616, - "40": 0.12646, - "41": 0.12693, - "42": 0.12768, - "43": 0.1269, - "44": 0.12617, - "45": 0.13224, - "46": 0.127, - "47": 0.12744, - "48": 0.12686, - "49": 0.12762, - "50": 0.1283, - "51": 0.17128, - "52": 0.14728, - "53": 0.12912, - "54": 0.1264, - "55": 0.12311, - "56": 0.12165, - "57": 0.1215, - "58": 0.12102, - "59": 0.12058, - "60": 0.1197, - "61": 0.1201, - "62": 0.11994, - "63": 0.12062, - "64": 0.12502, - "65": 0.12135, - "66": 0.12049, - "67": 0.11994, - "68": 0.12112, - "69": 0.12052, - "70": 0.12295, - "71": 0.11968, - "72": 0.11878, - "73": 0.12087, - "74": 0.11869, - "75": 0.1189, - "76": 0.12024, - "77": 0.11901, - "78": 0.11968, - "79": 0.11774, - "80": 0.11722, - "81": 0.11826, - "82": 0.11777, - "83": 0.11878, - "84": 0.11803, - "85": 0.11918, - "86": 0.11883, - "87": 0.11808, - "88": 0.11756, - "89": 0.11874, - "90": 0.11921, - "91": 0.11788, - "92": 0.12112, - "93": 0.11795, - "94": 0.11765, - "95": 0.12368, - "96": 0.11779, - "97": 0.11789, - "98": 0.11784, - "99": 0.11675, - "100": 0.11781 + "2": 5.72562, + "3": 0.12302, + "4": 0.12421, + "5": 0.1218, + "6": 0.12123, + "7": 0.12147, + "8": 0.12092, + "9": 0.12258, + "10": 0.12294, + "11": 0.12181, + "12": 0.1218, + "13": 0.12055, + "14": 0.12088, + "15": 0.12079, + "16": 0.12174, + "17": 0.12112, + "18": 0.12066, + "19": 0.12115, + "20": 0.12151, + "21": 0.12106, + "22": 0.12054, + "23": 0.12192, + "24": 0.1245, + "25": 0.12154, + "26": 0.12156, + "27": 0.12672, + "28": 0.12192, + "29": 0.12134, + "30": 0.12205, + "31": 0.1235, + "32": 0.12222, + "33": 0.12142, + "34": 0.12365, + "35": 0.1205, + "36": 0.1213, + "37": 0.12315, + "38": 0.1225, + "39": 0.12386, + "40": 0.12056, + "41": 0.1215, + "42": 0.1209, + "43": 0.12054, + "44": 0.1213, + "45": 0.12264, + "46": 0.12291, + "47": 0.1217, + "48": 0.1222, + "49": 0.12142, + "50": 0.1219, + "51": 0.26227, + "52": 0.15233, + "53": 0.12815, + "54": 0.12062, + "55": 0.1213, + "56": 0.12077, + "57": 0.12272, + "58": 0.12076, + "59": 0.12239, + "60": 0.12145, + "61": 0.11983, + "62": 0.11999, + "63": 0.12021, + "64": 0.11885, + "65": 0.11967, + "66": 0.11923, + "67": 0.12028, + "68": 0.12012, + "69": 0.11894, + "70": 0.12065, + "71": 0.12021, + "72": 0.11949, + "73": 0.11998, + "74": 0.11954, + "75": 0.12081, + "76": 0.11959, + "77": 0.12003, + "78": 0.11974, + "79": 0.11965, + "80": 0.11945, + "81": 0.12092, + "82": 0.12172, + "83": 0.12076, + "84": 0.12018, + "85": 0.12039, + "86": 0.12127, + "87": 0.1214, + "88": 0.12047, + "89": 0.11985, + "90": 0.12319, + "91": 0.12061, + "92": 0.12047, + "93": 0.12169, + "94": 0.12001, + "95": 0.12027, + "96": 0.11824, + "97": 0.11873, + "98": 0.11964, + "99": 0.11983, + "100": 0.11983 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json index 4476edef5ff..78c53757a24 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92753, - "2": 10.92559, + "1": 10.92752, + "2": 10.92554, "3": 10.93114, - "4": 10.93741, - "5": 10.93016, - "6": 10.92355, - "7": 10.93863, - "8": 10.92774, - "9": 10.93076, - "10": 10.92802, - "11": 10.91411, - "12": 10.92298, - "13": 10.92014, - "14": 10.90614, + "4": 10.93727, + "5": 10.93015, + "6": 10.92346, + "7": 10.93864, + "8": 10.92777, + "9": 10.93074, + "10": 10.92801, + "11": 10.91409, + "12": 10.92299, + "13": 10.92006, + "14": 10.90618, "15": 10.88338, - "16": 10.87951, - "17": 10.88868, - "18": 10.86451, - "19": 10.87355, - "20": 10.783, - "21": 10.78553, - "22": 10.76764, - "23": 10.764, - "24": 10.72663, - "25": 10.73401, - "26": 10.71265, - "27": 10.67569, - "28": 10.6012, - "29": 10.58219, - "30": 10.55702, - "31": 10.56167, - "32": 10.53487, - "33": 10.50348, - "34": 10.46525, - "35": 10.47887, - "36": 10.45601, - "37": 10.41808, - "38": 10.4167, - "39": 10.38178, - "40": 10.37615, - "41": 10.34328, - "42": 10.33161, - "43": 10.3077, - "44": 10.27713, - "45": 10.2988, - "46": 10.25974, - "47": 10.23655, - "48": 10.19378, - "49": 10.18543, - "50": 10.19728, - "51": 10.19367, - "52": 10.14697, - "53": 10.14982, - "54": 10.11733, - "55": 10.08476, - "56": 10.12202, - "57": 10.10466, - "58": 10.11902, - "59": 10.06425, - "60": 10.08087, + "16": 10.87953, + "17": 10.88861, + "18": 10.86454, + "19": 10.87347, + "20": 10.78318, + "21": 10.78551, + "22": 10.76768, + "23": 10.76394, + "24": 10.72672, + "25": 10.734, + "26": 10.71269, + "27": 10.6757, + "28": 10.60128, + "29": 10.5823, + "30": 10.55707, + "31": 10.56165, + "32": 10.53484, + "33": 10.50353, + "34": 10.46529, + "35": 10.47894, + "36": 10.45605, + "37": 10.41812, + "38": 10.41677, + "39": 10.38189, + "40": 10.37623, + "41": 10.34336, + "42": 10.33171, + "43": 10.30768, + "44": 10.27724, + "45": 10.29884, + "46": 10.2598, + "47": 10.23664, + "48": 10.1938, + "49": 10.18553, + "50": 10.19738, + "51": 10.1937, + "52": 10.14694, + "53": 10.1499, + "54": 10.11736, + "55": 10.08481, + "56": 10.12206, + "57": 10.10473, + "58": 10.11908, + "59": 10.06432, + "60": 10.08092, "61": 10.03845, - "62": 10.00507, - "63": 10.07872, - "64": 10.03202, - "65": 10.00346, - "66": 10.03491, - "67": 10.00571, - "68": 9.97032, - "69": 9.99153, - "70": 9.97866, - "71": 9.9998, - "72": 9.97531, - "73": 9.96566, - "74": 9.95983, + "62": 10.00515, + "63": 10.07878, + "64": 10.03206, + "65": 10.00349, + "66": 10.03495, + "67": 10.00575, + "68": 9.97033, + "69": 9.99157, + "70": 9.97869, + "71": 9.99986, + "72": 9.97535, + "73": 9.96568, + "74": 9.95986, "75": 9.92549, - "76": 9.96566, + "76": 9.96565, "77": 9.95606, - "78": 9.90446, - "79": 9.91127, - "80": 9.92478, - "81": 9.94432, - "82": 9.88863, - "83": 9.85093, - "84": 9.78899, - "85": 9.77531, - "86": 9.88013, + "78": 9.9045, + "79": 9.91133, + "80": 9.92487, + "81": 9.94438, + "82": 9.88871, + "83": 9.85105, + "84": 9.78905, + "85": 9.77533, + "86": 9.88023, "87": 9.91309, - "88": 9.88639, - "89": 9.82366, - "90": 9.81326, - "91": 9.82136, - "92": 9.81011, - "93": 9.75216, - "94": 9.82571, - "95": 9.81912, - "96": 9.8055, - "97": 9.7436, - "98": 9.77248, - "99": 9.81661, - "100": 9.71126 + "88": 9.88646, + "89": 9.8237, + "90": 9.81331, + "91": 9.82141, + "92": 9.81015, + "93": 9.75217, + "94": 9.82575, + "95": 9.8191, + "96": 9.80552, + "97": 9.74366, + "98": 9.7725, + "99": 9.8167, + "100": 9.71129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1770.0, - "2": 1703.0, - "3": 1799.0, - "4": 1823.0, - "5": 1737.0, - "6": 1733.0, + "1": 1845.0, + "2": 1697.0, + "3": 1692.0, + "4": 1714.0, + "5": 1786.0, + "6": 1732.0, "7": 1952.0, - "8": 1738.0, - "9": 1687.0, - "10": 1780.0, - "11": 1673.0, - "12": 1677.0, - "13": 1724.0, - "14": 1837.0, - "15": 1570.0, - "16": 1727.0, - "17": 1788.0, - "18": 1780.0, - "19": 1660.0, - "20": 1694.0, - "21": 1732.0, - "22": 1727.0, - "23": 1703.0, - "24": 1799.0, - "25": 1588.0, - "26": 1817.0, - "27": 1684.0, - "28": 1866.0, - "29": 1939.0, - "30": 1970.0, - "31": 1940.0, - "32": 1908.0, - "33": 1954.0, - "34": 2074.0, - "35": 2035.0, - "36": 1969.0, - "37": 2171.0, - "38": 2085.0, - "39": 2307.0, - "40": 2354.0, - "41": 2281.0, - "42": 1999.0, - "43": 2430.0, - "44": 2201.0, - "45": 2656.0, - "46": 2281.0, - "47": 2477.0, - "48": 2552.0, - "49": 2811.0, - "50": 2570.0, - "51": 2414.0, - "52": 2753.0, - "53": 2603.0, - "54": 2894.0, - "55": 2730.0, - "56": 2607.0, - "57": 2214.0, - "58": 3601.0, - "59": 2895.0, - "60": 2853.0, - "61": 2642.0, - "62": 3082.0, - "63": 3315.0, - "64": 3551.0, - "65": 2620.0, - "66": 3064.0, - "67": 3946.0, - "68": 3280.0, - "69": 2902.0, - "70": 3285.0, - "71": 3061.0, - "72": 2994.0, - "73": 3480.0, - "74": 3147.0, - "75": 3252.0, - "76": 3151.0, - "77": 3721.0, - "78": 3131.0, - "79": 3305.0, - "80": 3197.0, - "81": 3396.0, - "82": 3230.0, - "83": 3266.0, - "84": 2830.0, - "85": 2720.0, - "86": 3098.0, - "87": 2969.0, - "88": 3044.0, - "89": 3029.0, - "90": 3586.0, - "91": 3189.0, - "92": 2969.0, - "93": 2896.0, - "94": 3171.0, - "95": 3403.0, - "96": 3597.0, - "97": 3484.0, - "98": 3353.0, - "99": 3320.0, - "100": 3368.0 + "8": 1761.0, + "9": 1829.0, + "10": 1835.0, + "11": 1692.0, + "12": 1719.0, + "13": 1771.0, + "14": 1925.0, + "15": 1565.0, + "16": 1676.0, + "17": 1839.0, + "18": 1831.0, + "19": 1657.0, + "20": 1646.0, + "21": 1758.0, + "22": 1764.0, + "23": 1679.0, + "24": 1796.0, + "25": 1803.0, + "26": 1861.0, + "27": 1680.0, + "28": 1836.0, + "29": 1974.0, + "30": 1870.0, + "31": 1991.0, + "32": 1855.0, + "33": 1929.0, + "34": 1979.0, + "35": 2026.0, + "36": 1997.0, + "37": 2154.0, + "38": 2182.0, + "39": 2162.0, + "40": 2249.0, + "41": 2357.0, + "42": 1947.0, + "43": 2334.0, + "44": 2145.0, + "45": 2601.0, + "46": 2416.0, + "47": 2528.0, + "48": 2505.0, + "49": 2807.0, + "50": 2606.0, + "51": 2529.0, + "52": 2706.0, + "53": 2546.0, + "54": 2820.0, + "55": 2723.0, + "56": 2570.0, + "57": 2215.0, + "58": 3573.0, + "59": 2904.0, + "60": 2916.0, + "61": 2765.0, + "62": 3181.0, + "63": 3220.0, + "64": 3594.0, + "65": 2595.0, + "66": 3023.0, + "67": 3840.0, + "68": 3470.0, + "69": 2927.0, + "70": 3210.0, + "71": 3080.0, + "72": 2938.0, + "73": 3455.0, + "74": 3146.0, + "75": 3283.0, + "76": 3170.0, + "77": 3808.0, + "78": 3264.0, + "79": 3266.0, + "80": 3117.0, + "81": 3423.0, + "82": 3087.0, + "83": 3214.0, + "84": 2912.0, + "85": 2742.0, + "86": 3145.0, + "87": 2981.0, + "88": 3157.0, + "89": 2922.0, + "90": 3525.0, + "91": 3112.0, + "92": 3039.0, + "93": 3001.0, + "94": 3145.0, + "95": 3363.0, + "96": 3581.0, + "97": 3603.0, + "98": 3403.0, + "99": 3370.0, + "100": 3303.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.91781, - "3": 2.01209, - "4": 1.70703, - "5": 1.58348, - "6": 2.04178, - "7": 1.68383, - "8": 2.1805, - "9": 2.29254, - "10": 1.87808, - "11": 2.07274, - "12": 1.87306, - "13": 2.16607, - "14": 2.09193, - "15": 1.60699, - "16": 1.71658, - "17": 2.40105, - "18": 1.86098, - "19": 1.75259, - "20": 1.81546, - "21": 2.18524, - "22": 1.89571, - "23": 1.93775, - "24": 2.02166, - "25": 2.23446, - "26": 2.42493, - "27": 2.03299, - "28": 1.917, - "29": 1.68675, - "30": 1.97254, - "31": 1.5062, - "32": 2.99424, - "33": 1.6385, - "34": 1.692, - "35": 2.29559, - "36": 2.26179, - "37": 1.98836, - "38": 1.4767, - "39": 2.42606, - "40": 2.06154, - "41": 1.85116, - "42": 2.68553, - "43": 1.94445, - "44": 1.81498, - "45": 2.2976, - "46": 1.90665, - "47": 1.85997, - "48": 2.40369, - "49": 1.89519, - "50": 2.15372, - "51": 1.50287, - "52": 2.04698, - "53": 2.17394, - "54": 2.19961, - "55": 2.10794, - "56": 2.76028, - "57": 1.42194, - "58": 2.1181, - "59": 1.60096, - "60": 1.68702, - "61": 2.10997, - "62": 2.402, - "63": 1.76578, - "64": 2.04048, - "65": 1.92492, - "66": 2.09545, - "67": 2.18965, - "68": 1.41838, - "69": 2.07955, - "70": 2.01637, - "71": 2.36996, - "72": 1.86862, - "73": 1.90973, - "74": 1.92846, - "75": 1.62216, - "76": 1.87281, - "77": 1.84399, - "78": 1.69104, - "79": 2.26455, - "80": 1.95583, - "81": 2.16425, - "82": 2.33086, - "83": 2.02687, - "84": 1.78718, - "85": 2.01981, - "86": 1.77488, - "87": 2.50179, - "88": 2.17543, - "89": 2.25997, - "90": 1.99416, - "91": 2.10642, - "92": 2.65749, - "93": 1.71244, - "94": 2.00789, - "95": 2.13119, - "96": 1.93962, - "97": 1.87888, - "98": 2.17582, - "99": 2.72487, - "100": 1.95412 + "2": 5.04703, + "3": 1.99199, + "4": 2.09518, + "5": 1.58663, + "6": 1.99113, + "7": 1.45759, + "8": 2.33841, + "9": 2.36279, + "10": 1.87261, + "11": 2.13869, + "12": 1.92109, + "13": 2.10956, + "14": 1.98257, + "15": 1.303, + "16": 1.90815, + "17": 2.29532, + "18": 1.93725, + "19": 2.12302, + "20": 1.87664, + "21": 2.24503, + "22": 1.90587, + "23": 2.25722, + "24": 1.89072, + "25": 2.10088, + "26": 2.5103, + "27": 1.52903, + "28": 1.51566, + "29": 1.19658, + "30": 1.576, + "31": 1.41349, + "32": 1.98279, + "33": 1.38672, + "34": 1.30376, + "35": 1.8636, + "36": 1.46009, + "37": 1.26068, + "38": 1.20989, + "39": 1.73631, + "40": 1.71229, + "41": 1.51323, + "42": 2.23144, + "43": 1.66929, + "44": 1.82524, + "45": 2.29129, + "46": 1.77141, + "47": 1.69348, + "48": 1.85769, + "49": 1.6205, + "50": 2.04835, + "51": 1.0882, + "52": 2.28317, + "53": 1.97761, + "54": 2.18908, + "55": 1.78584, + "56": 2.49115, + "57": 1.31026, + "58": 1.92964, + "59": 1.86414, + "60": 1.53923, + "61": 2.43823, + "62": 2.09221, + "63": 1.48294, + "64": 1.8834, + "65": 1.48038, + "66": 1.90369, + "67": 1.52439, + "68": 1.38547, + "69": 1.9442, + "70": 1.60013, + "71": 2.24078, + "72": 1.65969, + "73": 1.48586, + "74": 1.05832, + "75": 1.53395, + "76": 1.99654, + "77": 1.41558, + "78": 1.44104, + "79": 2.14073, + "80": 1.64252, + "81": 2.02058, + "82": 1.9555, + "83": 1.87248, + "84": 1.2153, + "85": 1.64283, + "86": 1.27483, + "87": 1.7866, + "88": 1.41131, + "89": 1.96302, + "90": 1.6895, + "91": 1.77288, + "92": 2.23073, + "93": 1.75114, + "94": 1.90935, + "95": 2.10873, + "96": 2.15486, + "97": 1.76351, + "98": 2.20924, + "99": 2.45275, + "100": 1.71934 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json index d479cad094f..97393c2cd83 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.91715, - "2": 10.91532, - "3": 10.90493, + "2": 10.9154, + "3": 10.90491, "4": 10.90847, - "5": 10.90691, - "6": 10.9055, - "7": 10.92235, + "5": 10.90687, + "6": 10.90548, + "7": 10.92229, "8": 10.90215, - "9": 10.90498, - "10": 10.89605, - "11": 10.90018, - "12": 10.90163, + "9": 10.905, + "10": 10.89603, + "11": 10.90014, + "12": 10.90167, "13": 10.90201, - "14": 10.88651, - "15": 10.89684, - "16": 10.86925, - "17": 10.87976, - "18": 10.84695, - "19": 10.87292, - "20": 10.82075, - "21": 10.82485, - "22": 10.81217, - "23": 10.80616, - "24": 10.76755, - "25": 10.78042, - "26": 10.76321, - "27": 10.7656, - "28": 10.70637, - "29": 10.68618, - "30": 10.66655, - "31": 10.65301, - "32": 10.64628, - "33": 10.61261, - "34": 10.5899, - "35": 10.58928, - "36": 10.56372, - "37": 10.55028, - "38": 10.55748, + "14": 10.88648, + "15": 10.8968, + "16": 10.86928, + "17": 10.87979, + "18": 10.847, + "19": 10.87283, + "20": 10.82063, + "21": 10.82483, + "22": 10.81215, + "23": 10.80624, + "24": 10.76779, + "25": 10.78047, + "26": 10.76316, + "27": 10.76555, + "28": 10.70634, + "29": 10.68625, + "30": 10.66666, + "31": 10.65299, + "32": 10.64627, + "33": 10.61257, + "34": 10.58996, + "35": 10.58931, + "36": 10.5638, + "37": 10.5502, + "38": 10.5576, "39": 10.50946, - "40": 10.49801, - "41": 10.46839, - "42": 10.46037, - "43": 10.42389, - "44": 10.40559, - "45": 10.40957, - "46": 10.36962, - "47": 10.36339, - "48": 10.31729, - "49": 10.31454, - "50": 10.31019, - "51": 10.29764, - "52": 10.25606, - "53": 10.24571, - "54": 10.21517, - "55": 10.18762, + "40": 10.49804, + "41": 10.46843, + "42": 10.46039, + "43": 10.42395, + "44": 10.40554, + "45": 10.40958, + "46": 10.36961, + "47": 10.36341, + "48": 10.31738, + "49": 10.3146, + "50": 10.31027, + "51": 10.29769, + "52": 10.25613, + "53": 10.24574, + "54": 10.21521, + "55": 10.1877, "56": 10.21176, - "57": 10.1873, - "58": 10.20309, - "59": 10.15361, - "60": 10.15374, - "61": 10.12073, - "62": 10.07994, - "63": 10.14838, - "64": 10.09518, - "65": 10.06418, - "66": 10.08944, - "67": 10.06057, - "68": 10.02184, - "69": 10.03115, - "70": 10.02659, - "71": 10.0429, - "72": 10.0198, - "73": 9.99838, - "74": 9.98812, - "75": 9.9503, - "76": 10.00097, - "77": 9.99194, + "57": 10.18736, + "58": 10.20312, + "59": 10.15362, + "60": 10.15372, + "61": 10.12077, + "62": 10.07997, + "63": 10.14849, + "64": 10.09525, + "65": 10.06421, + "66": 10.08945, + "67": 10.06061, + "68": 10.02194, + "69": 10.03116, + "70": 10.02658, + "71": 10.04295, + "72": 10.01976, + "73": 9.99839, + "74": 9.98819, + "75": 9.95031, + "76": 10.00101, + "77": 9.992, "78": 9.93894, - "79": 9.9475, - "80": 9.95199, - "81": 9.97687, - "82": 9.93072, - "83": 9.89056, - "84": 9.82765, - "85": 9.81182, - "86": 9.9222, - "87": 9.94021, - "88": 9.9185, - "89": 9.85561, - "90": 9.85341, - "91": 9.86676, - "92": 9.85465, - "93": 9.79188, - "94": 9.86491, - "95": 9.86283, - "96": 9.84804, - "97": 9.78916, - "98": 9.82351, - "99": 9.86502, - "100": 9.76301 + "79": 9.94753, + "80": 9.95206, + "81": 9.97693, + "82": 9.93076, + "83": 9.89058, + "84": 9.82768, + "85": 9.81185, + "86": 9.92221, + "87": 9.94022, + "88": 9.91853, + "89": 9.8556, + "90": 9.85347, + "91": 9.8667, + "92": 9.85471, + "93": 9.79187, + "94": 9.8649, + "95": 9.86278, + "96": 9.8481, + "97": 9.78919, + "98": 9.82356, + "99": 9.86505, + "100": 9.76305 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2974.0, - "2": 3007.0, - "3": 2774.0, - "4": 2880.0, - "5": 2917.0, - "6": 2797.0, - "7": 2954.0, - "8": 2881.0, - "9": 3016.0, - "10": 3002.0, - "11": 3025.0, - "12": 2876.0, - "13": 3098.0, - "14": 3056.0, - "15": 2793.0, - "16": 2980.0, - "17": 2895.0, - "18": 2749.0, - "19": 3152.0, - "20": 2925.0, - "21": 2901.0, - "22": 2845.0, - "23": 2802.0, - "24": 2832.0, - "25": 3001.0, - "26": 2853.0, - "27": 2791.0, - "28": 2843.0, - "29": 2785.0, - "30": 2711.0, - "31": 2699.0, - "32": 2647.0, - "33": 2622.0, - "34": 2823.0, - "35": 2677.0, - "36": 2661.0, - "37": 2791.0, - "38": 2749.0, - "39": 2543.0, - "40": 2735.0, - "41": 2547.0, - "42": 2688.0, - "43": 2594.0, - "44": 2635.0, - "45": 2613.0, - "46": 2635.0, - "47": 2571.0, - "48": 2605.0, - "49": 2710.0, - "50": 2619.0, - "51": 2728.0, - "52": 2683.0, - "53": 2654.0, - "54": 2822.0, - "55": 2512.0, - "56": 2942.0, - "57": 2461.0, - "58": 3258.0, - "59": 2858.0, - "60": 2770.0, - "61": 2863.0, - "62": 2811.0, - "63": 3004.0, - "64": 3008.0, - "65": 2584.0, - "66": 2961.0, - "67": 3223.0, - "68": 2963.0, - "69": 3022.0, - "70": 2992.0, - "71": 2761.0, - "72": 3101.0, - "73": 3109.0, - "74": 2917.0, - "75": 2949.0, - "76": 3185.0, - "77": 3265.0, - "78": 3091.0, - "79": 3067.0, - "80": 3041.0, - "81": 3235.0, - "82": 3487.0, - "83": 3187.0, - "84": 3089.0, - "85": 2761.0, - "86": 3019.0, - "87": 3151.0, - "88": 3283.0, - "89": 3203.0, - "90": 3469.0, - "91": 2839.0, - "92": 3056.0, - "93": 3055.0, - "94": 3245.0, - "95": 3213.0, - "96": 3280.0, - "97": 3508.0, - "98": 3586.0, - "99": 3138.0, - "100": 3337.0 + "1": 2918.0, + "2": 3102.0, + "3": 2838.0, + "4": 3016.0, + "5": 2866.0, + "6": 2736.0, + "7": 2941.0, + "8": 2853.0, + "9": 3050.0, + "10": 3021.0, + "11": 3044.0, + "12": 2869.0, + "13": 3184.0, + "14": 3101.0, + "15": 2827.0, + "16": 3146.0, + "17": 2960.0, + "18": 2676.0, + "19": 3031.0, + "20": 2898.0, + "21": 2961.0, + "22": 2898.0, + "23": 2698.0, + "24": 2874.0, + "25": 3055.0, + "26": 2787.0, + "27": 2867.0, + "28": 2832.0, + "29": 2835.0, + "30": 2845.0, + "31": 2598.0, + "32": 2709.0, + "33": 2551.0, + "34": 2674.0, + "35": 2666.0, + "36": 2641.0, + "37": 2633.0, + "38": 2709.0, + "39": 2502.0, + "40": 2702.0, + "41": 2519.0, + "42": 2662.0, + "43": 2706.0, + "44": 2529.0, + "45": 2729.0, + "46": 2625.0, + "47": 2637.0, + "48": 2644.0, + "49": 2640.0, + "50": 2512.0, + "51": 2810.0, + "52": 2735.0, + "53": 2596.0, + "54": 2751.0, + "55": 2550.0, + "56": 2895.0, + "57": 2555.0, + "58": 3169.0, + "59": 2771.0, + "60": 2744.0, + "61": 2896.0, + "62": 2917.0, + "63": 3047.0, + "64": 3092.0, + "65": 2659.0, + "66": 2953.0, + "67": 3177.0, + "68": 3018.0, + "69": 3070.0, + "70": 3010.0, + "71": 2935.0, + "72": 3099.0, + "73": 3172.0, + "74": 3012.0, + "75": 2879.0, + "76": 3272.0, + "77": 3240.0, + "78": 3120.0, + "79": 3224.0, + "80": 2985.0, + "81": 3310.0, + "82": 3405.0, + "83": 3033.0, + "84": 3023.0, + "85": 2821.0, + "86": 2995.0, + "87": 3146.0, + "88": 3236.0, + "89": 3216.0, + "90": 3396.0, + "91": 2877.0, + "92": 3067.0, + "93": 3052.0, + "94": 3194.0, + "95": 3059.0, + "96": 3288.0, + "97": 3526.0, + "98": 3653.0, + "99": 3087.0, + "100": 3208.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.68703, - "3": 2.27607, - "4": 1.95356, - "5": 1.43808, - "6": 1.66974, - "7": 1.82073, - "8": 2.45573, - "9": 2.45608, - "10": 1.88881, - "11": 2.2115, - "12": 1.76382, - "13": 1.7887, - "14": 1.94066, - "15": 1.87084, - "16": 1.8839, - "17": 2.15134, - "18": 1.87321, - "19": 2.00269, - "20": 1.8935, - "21": 2.23123, - "22": 1.68075, - "23": 2.2295, - "24": 1.89249, - "25": 2.04549, - "26": 2.23751, - "27": 1.97092, - "28": 1.87329, - "29": 1.65388, - "30": 1.64312, - "31": 1.58189, - "32": 2.71233, - "33": 1.54317, - "34": 1.52133, - "35": 2.36052, - "36": 2.12589, - "37": 1.87697, - "38": 1.68171, - "39": 2.02993, - "40": 1.85575, - "41": 1.76289, - "42": 2.56863, - "43": 1.98611, - "44": 1.60222, - "45": 2.34358, - "46": 1.89547, - "47": 1.77595, - "48": 2.16868, - "49": 1.64316, - "50": 2.07621, - "51": 0.91935, - "52": 2.4302, - "53": 1.77192, - "54": 1.93281, - "55": 1.75761, - "56": 2.85699, - "57": 1.44381, - "58": 2.09848, - "59": 1.81572, - "60": 1.71216, - "61": 2.07107, - "62": 2.19075, - "63": 1.54832, - "64": 1.95479, - "65": 1.68437, - "66": 2.35293, - "67": 2.04743, - "68": 1.54269, - "69": 1.95756, - "70": 2.02158, - "71": 2.22677, - "72": 2.15266, - "73": 1.74043, - "74": 1.65764, - "75": 1.73174, - "76": 2.24875, - "77": 1.71524, - "78": 1.81966, - "79": 2.05485, - "80": 1.99433, - "81": 2.21734, - "82": 2.06341, - "83": 2.00711, - "84": 1.65729, - "85": 1.88703, - "86": 1.77847, - "87": 2.37983, - "88": 1.89045, - "89": 2.40014, - "90": 2.14625, - "91": 2.02425, - "92": 2.18983, - "93": 1.43472, - "94": 1.92868, - "95": 1.85937, - "96": 2.18236, - "97": 1.76772, - "98": 1.93071, - "99": 2.37598, - "100": 1.72405 + "2": 7.21955, + "3": 1.92976, + "4": 1.4823, + "5": 1.39012, + "6": 1.72529, + "7": 1.46228, + "8": 2.24496, + "9": 1.90135, + "10": 1.84282, + "11": 1.60582, + "12": 1.42457, + "13": 1.69316, + "14": 1.98382, + "15": 1.14211, + "16": 1.51999, + "17": 1.89552, + "18": 1.78664, + "19": 1.58933, + "20": 1.62929, + "21": 1.79955, + "22": 1.49806, + "23": 1.77747, + "24": 1.21202, + "25": 1.9506, + "26": 1.77203, + "27": 1.72278, + "28": 2.6689, + "29": 1.2642, + "30": 1.53421, + "31": 1.33283, + "32": 2.19391, + "33": 1.139, + "34": 1.34638, + "35": 1.78889, + "36": 1.74564, + "37": 1.3101, + "38": 1.16831, + "39": 1.39351, + "40": 1.62667, + "41": 1.57793, + "42": 2.02858, + "43": 1.68248, + "44": 1.5161, + "45": 2.33859, + "46": 1.48081, + "47": 1.54736, + "48": 1.54624, + "49": 1.2045, + "50": 1.79195, + "51": 1.06763, + "52": 1.61026, + "53": 1.70816, + "54": 1.82187, + "55": 1.5851, + "56": 1.57524, + "57": 0.9167, + "58": 1.51522, + "59": 1.21513, + "60": 1.21143, + "61": 1.70028, + "62": 1.76801, + "63": 1.30403, + "64": 1.5466, + "65": 1.31385, + "66": 1.39513, + "67": 1.3431, + "68": 1.26841, + "69": 1.89417, + "70": 1.5377, + "71": 2.27476, + "72": 1.74485, + "73": 1.90799, + "74": 1.17096, + "75": 1.78036, + "76": 1.95566, + "77": 1.86097, + "78": 1.81932, + "79": 2.59845, + "80": 1.4529, + "81": 1.66966, + "82": 1.69646, + "83": 1.94799, + "84": 1.5062, + "85": 1.74404, + "86": 1.5297, + "87": 1.95766, + "88": 2.27667, + "89": 1.74339, + "90": 1.77631, + "91": 1.60111, + "92": 2.16551, + "93": 1.6345, + "94": 1.9475, + "95": 2.17504, + "96": 2.04189, + "97": 1.87112, + "98": 1.83799, + "99": 2.66683, + "100": 1.53947 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json index 68f212b9a01..e311adf5f42 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9543, - "2": 10.97008, - "3": 10.96655, - "4": 10.96406, - "5": 10.96112, - "6": 10.95417, - "7": 10.9563, - "8": 10.95685, + "1": 10.95431, + "2": 10.97013, + "3": 10.96654, + "4": 10.96407, + "5": 10.96107, + "6": 10.9542, + "7": 10.95647, + "8": 10.95687, "9": 10.96029, - "10": 10.95842, - "11": 10.95613, - "12": 10.93265, - "13": 10.95939, - "14": 10.94942, - "15": 10.92043, - "16": 10.9, - "17": 10.9186, - "18": 10.90008, - "19": 10.91326, - "20": 10.8208, - "21": 10.81358, - "22": 10.80493, - "23": 10.78963, - "24": 10.75475, - "25": 10.75922, - "26": 10.7484, - "27": 10.69712, - "28": 10.6374, - "29": 10.61525, - "30": 10.5879, - "31": 10.5788, - "32": 10.55825, - "33": 10.52366, - "34": 10.50297, + "10": 10.95843, + "11": 10.95616, + "12": 10.93269, + "13": 10.95941, + "14": 10.94943, + "15": 10.92042, + "16": 10.89999, + "17": 10.91871, + "18": 10.90007, + "19": 10.91327, + "20": 10.82088, + "21": 10.81366, + "22": 10.80501, + "23": 10.78962, + "24": 10.75486, + "25": 10.75928, + "26": 10.74825, + "27": 10.69711, + "28": 10.63743, + "29": 10.61526, + "30": 10.58797, + "31": 10.57883, + "32": 10.5583, + "33": 10.52369, + "34": 10.50303, "35": 10.4836, - "36": 10.46154, - "37": 10.45411, - "38": 10.45226, - "39": 10.40934, - "40": 10.40241, - "41": 10.37868, - "42": 10.32566, - "43": 10.3166, - "44": 10.27992, - "45": 10.30832, - "46": 10.2472, - "47": 10.23867, - "48": 10.22249, - "49": 10.20589, - "50": 10.18444, - "51": 10.19259, - "52": 10.15913, - "53": 10.16844, - "54": 10.11266, - "55": 10.08473, - "56": 10.10193, - "57": 10.1234, + "36": 10.4616, + "37": 10.4541, + "38": 10.45224, + "39": 10.40943, + "40": 10.40249, + "41": 10.37869, + "42": 10.32564, + "43": 10.31674, + "44": 10.27993, + "45": 10.30843, + "46": 10.24724, + "47": 10.23875, + "48": 10.22256, + "49": 10.20598, + "50": 10.1845, + "51": 10.19265, + "52": 10.15917, + "53": 10.1685, + "54": 10.11269, + "55": 10.08475, + "56": 10.10202, + "57": 10.12348, "58": 10.10239, - "59": 10.05154, - "60": 10.08328, - "61": 10.02215, - "62": 9.99326, - "63": 10.06757, - "64": 10.02651, - "65": 10.02787, - "66": 10.02544, - "67": 9.99694, - "68": 9.97259, - "69": 10.02185, - "70": 9.98222, - "71": 9.98457, - "72": 10.00073, - "73": 9.96104, - "74": 9.95153, - "75": 9.9143, - "76": 9.95246, - "77": 9.94443, - "78": 9.9106, - "79": 9.90321, - "80": 9.91084, - "81": 9.93261, - "82": 9.86692, - "83": 9.84165, - "84": 9.79596, - "85": 9.76662, - "86": 9.88041, - "87": 9.86768, - "88": 9.86223, - "89": 9.80258, - "90": 9.78297, - "91": 9.82699, - "92": 9.79113, - "93": 9.71999, - "94": 9.81833, - "95": 9.79024, - "96": 9.81142, - "97": 9.74197, - "98": 9.76131, - "99": 9.79043, - "100": 9.70982 + "59": 10.0516, + "60": 10.08336, + "61": 10.0222, + "62": 9.99328, + "63": 10.06761, + "64": 10.02659, + "65": 10.0279, + "66": 10.02551, + "67": 9.99697, + "68": 9.9726, + "69": 10.02186, + "70": 9.98233, + "71": 9.98462, + "72": 10.00081, + "73": 9.9611, + "74": 9.95154, + "75": 9.91439, + "76": 9.9525, + "77": 9.94444, + "78": 9.91065, + "79": 9.90324, + "80": 9.91086, + "81": 9.93267, + "82": 9.86695, + "83": 9.84171, + "84": 9.79603, + "85": 9.76671, + "86": 9.88046, + "87": 9.86776, + "88": 9.86231, + "89": 9.8027, + "90": 9.78301, + "91": 9.827, + "92": 9.79116, + "93": 9.72008, + "94": 9.81837, + "95": 9.7903, + "96": 9.81145, + "97": 9.74202, + "98": 9.76135, + "99": 9.79047, + "100": 9.70983 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22962222.0, - "2": 22849884.0, - "3": 22709196.0, - "4": 22778684.0, - "5": 22783510.0, - "6": 22746664.0, - "7": 22872212.0, - "8": 22611268.0, - "9": 22757986.0, - "10": 22489102.0, - "11": 22753568.0, - "12": 22640822.0, + "1": 22962356.0, + "2": 22849760.0, + "3": 22709136.0, + "4": 22778680.0, + "5": 22783668.0, + "6": 22746670.0, + "7": 22872268.0, + "8": 22611380.0, + "9": 22757932.0, + "10": 22489028.0, + "11": 22753676.0, + "12": 22640800.0, "13": 23322448.0, - "14": 22991720.0, - "15": 22717780.0, - "16": 22822716.0, - "17": 22929916.0, - "18": 22994544.0, - "19": 23084352.0, - "20": 22722842.0, - "21": 22919028.0, - "22": 22944564.0, - "23": 22628360.0, - "24": 22862176.0, - "25": 22636342.0, - "26": 23004756.0, - "27": 22802356.0, - "28": 22998960.0, - "29": 22979398.0, - "30": 22943856.0, - "31": 22907830.0, - "32": 22665828.0, - "33": 22740540.0, - "34": 23075828.0, - "35": 22751008.0, + "14": 22991658.0, + "15": 22717790.0, + "16": 22822774.0, + "17": 22929936.0, + "18": 22994672.0, + "19": 23084308.0, + "20": 22722826.0, + "21": 22918960.0, + "22": 22944572.0, + "23": 22628440.0, + "24": 22862080.0, + "25": 22636290.0, + "26": 23004680.0, + "27": 22802366.0, + "28": 22998918.0, + "29": 22979550.0, + "30": 22943904.0, + "31": 22907900.0, + "32": 22665848.0, + "33": 22740528.0, + "34": 23075928.0, + "35": 22751018.0, "36": 22695452.0, - "37": 23103496.0, - "38": 22963920.0, - "39": 22984932.0, - "40": 22749492.0, - "41": 23064274.0, - "42": 22688540.0, - "43": 22987284.0, - "44": 22705372.0, - "45": 22847172.0, - "46": 22733438.0, - "47": 22849270.0, - "48": 22833516.0, - "49": 22888964.0, - "50": 22644492.0, - "51": 22699114.0, - "52": 22813308.0, - "53": 22961874.0, - "54": 22786016.0, - "55": 22922864.0, - "56": 22660044.0, - "57": 23193390.0, - "58": 22689138.0, - "59": 22842072.0, - "60": 23013902.0, - "61": 22672740.0, - "62": 22732050.0, - "63": 22632330.0, - "64": 22998972.0, - "65": 23198328.0, - "66": 22694544.0, - "67": 22955768.0, - "68": 22924760.0, - "69": 23149964.0, - "70": 22821652.0, - "71": 22739272.0, - "72": 23116444.0, - "73": 23133048.0, - "74": 22941152.0, - "75": 22880638.0, - "76": 22702690.0, - "77": 22980352.0, - "78": 22977154.0, - "79": 22824012.0, - "80": 22931284.0, - "81": 22834754.0, - "82": 22730492.0, - "83": 22735268.0, - "84": 23099376.0, - "85": 22922288.0, - "86": 23068188.0, - "87": 22373858.0, - "88": 22555568.0, - "89": 22727848.0, - "90": 22766116.0, - "91": 22919320.0, - "92": 22672776.0, - "93": 22646124.0, - "94": 23128378.0, - "95": 22694986.0, - "96": 22844902.0, - "97": 22831948.0, - "98": 22875216.0, - "99": 22636834.0, - "100": 23000010.0 + "37": 23103392.0, + "38": 22963982.0, + "39": 22984994.0, + "40": 22749560.0, + "41": 23064300.0, + "42": 22688442.0, + "43": 22987388.0, + "44": 22705344.0, + "45": 22847152.0, + "46": 22733516.0, + "47": 22849348.0, + "48": 22833504.0, + "49": 22888996.0, + "50": 22644588.0, + "51": 22698992.0, + "52": 22813148.0, + "53": 22961906.0, + "54": 22786102.0, + "55": 22922978.0, + "56": 22659918.0, + "57": 23193362.0, + "58": 22689076.0, + "59": 22841974.0, + "60": 23013940.0, + "61": 22672824.0, + "62": 22731932.0, + "63": 22632314.0, + "64": 22998856.0, + "65": 23198306.0, + "66": 22694492.0, + "67": 22955878.0, + "68": 22924802.0, + "69": 23150054.0, + "70": 22821700.0, + "71": 22739400.0, + "72": 23116542.0, + "73": 23133058.0, + "74": 22941244.0, + "75": 22880492.0, + "76": 22702724.0, + "77": 22980612.0, + "78": 22977188.0, + "79": 22823984.0, + "80": 22931380.0, + "81": 22834816.0, + "82": 22730594.0, + "83": 22735228.0, + "84": 23099324.0, + "85": 22922364.0, + "86": 23068054.0, + "87": 22373852.0, + "88": 22555668.0, + "89": 22727868.0, + "90": 22766180.0, + "91": 22919328.0, + "92": 22672822.0, + "93": 22646240.0, + "94": 23128352.0, + "95": 22694976.0, + "96": 22845072.0, + "97": 22832008.0, + "98": 22875316.0, + "99": 22636810.0, + "100": 22999952.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.06947, - "3": 0.1325, - "4": 0.13148, - "5": 0.13399, - "6": 0.1307, - "7": 0.12914, - "8": 0.12961, - "9": 0.13237, - "10": 0.12969, - "11": 0.13307, - "12": 0.13058, - "13": 0.13359, - "14": 0.12993, - "15": 0.12945, - "16": 0.13047, - "17": 0.13018, - "18": 0.13138, - "19": 0.13287, - "20": 0.13272, - "21": 0.12973, - "22": 0.12966, - "23": 0.12885, - "24": 0.12915, - "25": 0.13099, - "26": 0.13074, - "27": 0.13092, - "28": 0.1292, - "29": 0.12943, - "30": 0.12915, - "31": 0.13064, - "32": 0.12875, - "33": 0.1288, - "34": 0.12897, - "35": 0.13008, - "36": 0.12938, - "37": 0.12887, - "38": 0.1292, - "39": 0.13043, - "40": 0.1299, - "41": 0.13058, - "42": 0.13287, - "43": 0.13083, - "44": 0.13041, - "45": 0.13029, - "46": 0.13071, - "47": 0.12945, - "48": 0.13094, - "49": 0.13019, - "50": 0.12821, - "51": 0.17187, - "52": 0.16994, - "53": 0.14122, - "54": 0.13376, - "55": 0.1306, - "56": 0.12956, - "57": 0.13098, - "58": 0.12962, - "59": 0.13213, - "60": 0.13079, - "61": 0.13249, - "62": 0.13214, - "63": 0.13059, - "64": 0.12978, - "65": 0.13175, - "66": 0.12867, - "67": 0.13339, - "68": 0.13442, - "69": 0.13091, - "70": 0.13231, - "71": 0.12906, - "72": 0.12935, - "73": 0.12996, - "74": 0.12895, - "75": 0.13049, - "76": 0.13039, - "77": 0.13206, - "78": 0.1302, - "79": 0.12975, - "80": 0.13035, - "81": 0.13017, - "82": 0.13307, - "83": 0.13048, - "84": 0.1336, - "85": 0.13085, - "86": 0.12931, - "87": 0.12852, - "88": 0.13013, - "89": 0.1294, - "90": 0.13005, - "91": 0.12987, - "92": 0.13082, - "93": 0.13156, - "94": 0.12954, - "95": 0.12818, - "96": 0.13009, - "97": 0.12906, - "98": 0.12838, - "99": 0.12875, - "100": 0.12869 + "2": 6.66184, + "3": 0.14329, + "4": 0.13671, + "5": 0.13673, + "6": 0.13688, + "7": 0.13712, + "8": 0.13623, + "9": 0.13798, + "10": 0.13697, + "11": 0.13646, + "12": 0.13616, + "13": 0.13591, + "14": 0.1457, + "15": 0.13692, + "16": 0.13932, + "17": 0.13662, + "18": 0.13691, + "19": 0.13639, + "20": 0.1366, + "21": 0.13596, + "22": 0.13713, + "23": 0.13605, + "24": 0.13679, + "25": 0.13705, + "26": 0.13713, + "27": 0.13845, + "28": 0.13809, + "29": 0.13605, + "30": 0.13975, + "31": 0.13607, + "32": 0.13493, + "33": 0.13555, + "34": 0.13535, + "35": 0.13492, + "36": 0.13633, + "37": 0.13543, + "38": 0.13501, + "39": 0.13611, + "40": 0.13668, + "41": 0.13847, + "42": 0.13636, + "43": 0.13573, + "44": 0.13586, + "45": 0.13586, + "46": 0.13609, + "47": 0.13592, + "48": 0.13586, + "49": 0.13788, + "50": 0.13713, + "51": 0.17802, + "52": 0.16732, + "53": 0.14356, + "54": 0.13882, + "55": 0.13724, + "56": 0.13724, + "57": 0.13695, + "58": 0.13739, + "59": 0.13688, + "60": 0.13604, + "61": 0.13522, + "62": 0.13512, + "63": 0.13509, + "64": 0.13803, + "65": 0.13699, + "66": 0.13548, + "67": 0.13619, + "68": 0.13667, + "69": 0.13626, + "70": 0.13856, + "71": 0.13692, + "72": 0.1369, + "73": 0.13725, + "74": 0.13729, + "75": 0.13751, + "76": 0.13574, + "77": 0.13744, + "78": 0.13709, + "79": 0.13664, + "80": 0.13528, + "81": 0.13594, + "82": 0.1366, + "83": 0.13771, + "84": 0.13726, + "85": 0.14223, + "86": 0.13636, + "87": 0.13521, + "88": 0.13628, + "89": 0.13577, + "90": 0.13714, + "91": 0.13469, + "92": 0.13453, + "93": 0.13428, + "94": 0.13482, + "95": 0.13452, + "96": 0.13596, + "97": 0.13461, + "98": 0.13563, + "99": 0.13575, + "100": 0.13514 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json index 6db0ba1d33f..ec8aabce360 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.94455, - "2": 10.94743, - "3": 10.94818, - "4": 10.94206, - "5": 10.95026, - "6": 10.94824, - "7": 10.93493, - "8": 10.94753, - "9": 10.93762, - "10": 10.94101, - "11": 10.93294, - "12": 10.94364, - "13": 10.90622, - "14": 10.9172, - "15": 10.89665, - "16": 10.89474, - "17": 10.88785, - "18": 10.86864, - "19": 10.87541, - "20": 10.81655, - "21": 10.78013, - "22": 10.77103, - "23": 10.76886, - "24": 10.74831, - "25": 10.74345, - "26": 10.72258, - "27": 10.6734, - "28": 10.60315, - "29": 10.57246, - "30": 10.56303, - "31": 10.54422, - "32": 10.53105, - "33": 10.48751, - "34": 10.47659, - "35": 10.45645, - "36": 10.44068, + "1": 10.94451, + "2": 10.94741, + "3": 10.94825, + "4": 10.94201, + "5": 10.95027, + "6": 10.94816, + "7": 10.93494, + "8": 10.94743, + "9": 10.93757, + "10": 10.94107, + "11": 10.93295, + "12": 10.94366, + "13": 10.90623, + "14": 10.91718, + "15": 10.89679, + "16": 10.89471, + "17": 10.88797, + "18": 10.86869, + "19": 10.8755, + "20": 10.81657, + "21": 10.78016, + "22": 10.77117, + "23": 10.76887, + "24": 10.74837, + "25": 10.74347, + "26": 10.72265, + "27": 10.67342, + "28": 10.60322, + "29": 10.57247, + "30": 10.56312, + "31": 10.54428, + "32": 10.53104, + "33": 10.48753, + "34": 10.47664, + "35": 10.45653, + "36": 10.44072, "37": 10.40765, - "38": 10.40511, - "39": 10.36532, - "40": 10.3672, - "41": 10.33527, - "42": 10.30512, - "43": 10.28352, - "44": 10.255, - "45": 10.2688, + "38": 10.40518, + "39": 10.3653, + "40": 10.3673, + "41": 10.33535, + "42": 10.30523, + "43": 10.28355, + "44": 10.25508, + "45": 10.26878, "46": 10.22196, - "47": 10.20823, - "48": 10.18878, - "49": 10.17404, - "50": 10.1681, - "51": 10.16394, - "52": 10.12751, - "53": 10.12716, - "54": 10.09558, - "55": 10.06465, - "56": 10.07494, - "57": 10.07258, - "58": 10.08235, - "59": 10.02218, - "60": 10.04248, - "61": 10.00366, - "62": 9.97281, - "63": 10.05249, - "64": 10.00735, - "65": 9.98001, + "47": 10.20826, + "48": 10.18885, + "49": 10.17403, + "50": 10.1682, + "51": 10.16398, + "52": 10.12763, + "53": 10.12719, + "54": 10.09562, + "55": 10.0647, + "56": 10.07497, + "57": 10.07271, + "58": 10.08236, + "59": 10.02224, + "60": 10.04256, + "61": 10.00376, + "62": 9.97284, + "63": 10.05248, + "64": 10.00736, + "65": 9.98009, "66": 9.99713, - "67": 9.96053, - "68": 9.93587, - "69": 9.9726, - "70": 9.94511, - "71": 9.96529, - "72": 9.94714, - "73": 9.92131, - "74": 9.91335, - "75": 9.88321, - "76": 9.92543, - "77": 9.90561, - "78": 9.86953, - "79": 9.86598, - "80": 9.87842, - "81": 9.89412, - "82": 9.82675, - "83": 9.80186, - "84": 9.74788, - "85": 9.72843, - "86": 9.84286, - "87": 9.85715, - "88": 9.83812, - "89": 9.77531, - "90": 9.76324, + "67": 9.96062, + "68": 9.93589, + "69": 9.97263, + "70": 9.94508, + "71": 9.96537, + "72": 9.94715, + "73": 9.92134, + "74": 9.91339, + "75": 9.8833, + "76": 9.92544, + "77": 9.90563, + "78": 9.86958, + "79": 9.86605, + "80": 9.87846, + "81": 9.89408, + "82": 9.82678, + "83": 9.80194, + "84": 9.74787, + "85": 9.72849, + "86": 9.84283, + "87": 9.85721, + "88": 9.83815, + "89": 9.77535, + "90": 9.76325, "91": 9.78325, - "92": 9.75571, - "93": 9.70188, - "94": 9.77557, - "95": 9.76597, - "96": 9.7691, - "97": 9.70025, - "98": 9.72317, - "99": 9.76326, - "100": 9.67042 + "92": 9.75581, + "93": 9.70192, + "94": 9.7756, + "95": 9.76601, + "96": 9.76916, + "97": 9.70033, + "98": 9.72326, + "99": 9.76327, + "100": 9.67049 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22961734.0, - "2": 22849292.0, - "3": 22709450.0, - "4": 22778272.0, - "5": 22783344.0, - "6": 22745688.0, - "7": 22871616.0, - "8": 22611788.0, - "9": 22758488.0, - "10": 22489664.0, - "11": 22753930.0, - "12": 22640498.0, - "13": 23322660.0, - "14": 22991940.0, - "15": 22718150.0, - "16": 22822544.0, - "17": 22929420.0, - "18": 22995424.0, + "1": 22961676.0, + "2": 22849324.0, + "3": 22709392.0, + "4": 22778208.0, + "5": 22783408.0, + "6": 22745682.0, + "7": 22871448.0, + "8": 22611742.0, + "9": 22758592.0, + "10": 22489788.0, + "11": 22754000.0, + "12": 22640580.0, + "13": 23322692.0, + "14": 22991980.0, + "15": 22718096.0, + "16": 22822408.0, + "17": 22929436.0, + "18": 22995432.0, "19": 23084828.0, - "20": 22723088.0, - "21": 22918220.0, - "22": 22945164.0, - "23": 22627484.0, - "24": 22862724.0, - "25": 22636220.0, - "26": 23004960.0, - "27": 22801468.0, - "28": 22999412.0, - "29": 22979382.0, - "30": 22943964.0, - "31": 22908184.0, - "32": 22665988.0, - "33": 22741296.0, - "34": 23076296.0, - "35": 22750520.0, - "36": 22695812.0, - "37": 23103140.0, - "38": 22964608.0, - "39": 22985704.0, - "40": 22749862.0, - "41": 23065512.0, - "42": 22687840.0, - "43": 22987484.0, - "44": 22704888.0, - "45": 22846928.0, - "46": 22734288.0, - "47": 22849852.0, - "48": 22833504.0, - "49": 22888668.0, - "50": 22644700.0, - "51": 22698624.0, - "52": 22812560.0, - "53": 22962504.0, - "54": 22786684.0, - "55": 22922392.0, - "56": 22660110.0, - "57": 23192904.0, - "58": 22688988.0, - "59": 22841898.0, - "60": 23013408.0, - "61": 22673272.0, - "62": 22732280.0, - "63": 22631808.0, - "64": 22999252.0, - "65": 23198302.0, - "66": 22694020.0, - "67": 22955048.0, - "68": 22924488.0, - "69": 23149376.0, - "70": 22822536.0, - "71": 22740440.0, - "72": 23117448.0, - "73": 23133530.0, - "74": 22941076.0, - "75": 22880362.0, - "76": 22702896.0, - "77": 22980008.0, - "78": 22977106.0, - "79": 22824536.0, - "80": 22930748.0, - "81": 22834604.0, - "82": 22731260.0, - "83": 22733720.0, - "84": 23099270.0, - "85": 22921714.0, - "86": 23069020.0, - "87": 22375336.0, - "88": 22555520.0, - "89": 22727918.0, - "90": 22765510.0, - "91": 22918544.0, - "92": 22672096.0, - "93": 22645440.0, - "94": 23128324.0, - "95": 22694938.0, - "96": 22845282.0, - "97": 22832220.0, - "98": 22874074.0, - "99": 22636662.0, - "100": 23000632.0 + "20": 22723064.0, + "21": 22918254.0, + "22": 22945122.0, + "23": 22627672.0, + "24": 22862716.0, + "25": 22636040.0, + "26": 23004826.0, + "27": 22801550.0, + "28": 22999472.0, + "29": 22979330.0, + "30": 22943888.0, + "31": 22908212.0, + "32": 22665978.0, + "33": 22741242.0, + "34": 23076366.0, + "35": 22750568.0, + "36": 22695876.0, + "37": 23103164.0, + "38": 22964606.0, + "39": 22985698.0, + "40": 22749752.0, + "41": 23065652.0, + "42": 22687862.0, + "43": 22987750.0, + "44": 22704962.0, + "45": 22846906.0, + "46": 22734100.0, + "47": 22849874.0, + "48": 22833476.0, + "49": 22888760.0, + "50": 22644576.0, + "51": 22698808.0, + "52": 22812564.0, + "53": 22962496.0, + "54": 22786708.0, + "55": 22922440.0, + "56": 22660224.0, + "57": 23192774.0, + "58": 22689076.0, + "59": 22841912.0, + "60": 23013384.0, + "61": 22673186.0, + "62": 22732284.0, + "63": 22631794.0, + "64": 22999306.0, + "65": 23198276.0, + "66": 22694010.0, + "67": 22955142.0, + "68": 22924460.0, + "69": 23149340.0, + "70": 22822656.0, + "71": 22740376.0, + "72": 23117308.0, + "73": 23133466.0, + "74": 22940986.0, + "75": 22880310.0, + "76": 22702712.0, + "77": 22979996.0, + "78": 22977068.0, + "79": 22824492.0, + "80": 22930828.0, + "81": 22834530.0, + "82": 22731320.0, + "83": 22733696.0, + "84": 23099368.0, + "85": 22921648.0, + "86": 23068952.0, + "87": 22375358.0, + "88": 22555496.0, + "89": 22727996.0, + "90": 22765896.0, + "91": 22918432.0, + "92": 22672088.0, + "93": 22645372.0, + "94": 23128346.0, + "95": 22694952.0, + "96": 22845202.0, + "97": 22832268.0, + "98": 22873974.0, + "99": 22636744.0, + "100": 23000626.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.15406, - "3": 2.8386, - "4": 2.51987, - "5": 2.04597, - "6": 2.28093, - "7": 1.89099, - "8": 2.46204, - "9": 2.52825, - "10": 2.19638, - "11": 2.69997, - "12": 1.90907, - "13": 2.47192, - "14": 2.66042, - "15": 2.13711, - "16": 2.21209, - "17": 2.92912, - "18": 2.52951, - "19": 2.23442, - "20": 2.20013, - "21": 2.52727, - "22": 1.85802, - "23": 2.38632, - "24": 2.23058, - "25": 2.59444, - "26": 2.57789, - "27": 2.27719, - "28": 2.34143, - "29": 1.92447, - "30": 1.99388, - "31": 1.73962, - "32": 3.35076, - "33": 1.82019, - "34": 1.91928, - "35": 2.69899, - "36": 2.49125, - "37": 2.29016, - "38": 1.65719, - "39": 2.95518, - "40": 2.55015, - "41": 2.13436, - "42": 3.33939, - "43": 2.54565, - "44": 3.25779, - "45": 2.60464, - "46": 2.12982, - "47": 2.88836, - "48": 2.64484, - "49": 2.10153, - "50": 2.56157, - "51": 1.96443, - "52": 2.49422, - "53": 1.91775, - "54": 2.69343, - "55": 2.5621, - "56": 2.81395, - "57": 1.77504, - "58": 2.57933, - "59": 2.17979, - "60": 2.18026, - "61": 2.83063, - "62": 3.02627, - "63": 2.08661, - "64": 2.23306, - "65": 2.21281, - "66": 2.79388, - "67": 2.24457, - "68": 1.82545, - "69": 2.36781, - "70": 2.3322, - "71": 2.63242, - "72": 2.20177, - "73": 2.3826, - "74": 2.17222, - "75": 2.19764, - "76": 2.33719, - "77": 2.17667, - "78": 2.15935, - "79": 2.67024, - "80": 2.04535, - "81": 2.45699, - "82": 2.5877, - "83": 2.1103, - "84": 2.31754, - "85": 3.84649, - "86": 2.31638, - "87": 2.86869, - "88": 2.90248, - "89": 3.28206, - "90": 2.45151, - "91": 2.54808, - "92": 3.15338, - "93": 2.26075, - "94": 2.85946, - "95": 2.7598, - "96": 2.08416, - "97": 2.13821, - "98": 2.27557, - "99": 2.67809, - "100": 2.27389 + "2": 6.19216, + "3": 1.96767, + "4": 1.65496, + "5": 1.31048, + "6": 2.21364, + "7": 1.34744, + "8": 2.06842, + "9": 2.13567, + "10": 1.49621, + "11": 1.99766, + "12": 1.48398, + "13": 1.58365, + "14": 2.07044, + "15": 1.61097, + "16": 1.91961, + "17": 2.29228, + "18": 1.88543, + "19": 1.7324, + "20": 1.96891, + "21": 2.00293, + "22": 1.43602, + "23": 2.1521, + "24": 1.84624, + "25": 2.29056, + "26": 2.28222, + "27": 1.55009, + "28": 1.8601, + "29": 1.29955, + "30": 1.63764, + "31": 1.16955, + "32": 2.30868, + "33": 1.16552, + "34": 1.33826, + "35": 1.77584, + "36": 1.72659, + "37": 1.89776, + "38": 1.44989, + "39": 1.93496, + "40": 1.60444, + "41": 1.65081, + "42": 2.32453, + "43": 1.54377, + "44": 1.53083, + "45": 2.03144, + "46": 1.27209, + "47": 1.2607, + "48": 1.70643, + "49": 1.40146, + "50": 1.62977, + "51": 1.49456, + "52": 1.69107, + "53": 1.84317, + "54": 2.00509, + "55": 1.62575, + "56": 1.79955, + "57": 1.26222, + "58": 1.92008, + "59": 1.46742, + "60": 1.45194, + "61": 2.32101, + "62": 1.97462, + "63": 1.43815, + "64": 1.65803, + "65": 1.54695, + "66": 1.92585, + "67": 1.67853, + "68": 1.16115, + "69": 1.48183, + "70": 1.37159, + "71": 2.08557, + "72": 1.31939, + "73": 1.56483, + "74": 1.36384, + "75": 1.58991, + "76": 1.7661, + "77": 1.54763, + "78": 1.63629, + "79": 1.95756, + "80": 1.77042, + "81": 1.62117, + "82": 2.07593, + "83": 1.91434, + "84": 1.02049, + "85": 1.4322, + "86": 1.3025, + "87": 1.91189, + "88": 1.50973, + "89": 1.96332, + "90": 1.59436, + "91": 1.84354, + "92": 2.07359, + "93": 1.49324, + "94": 1.69648, + "95": 1.50551, + "96": 1.79982, + "97": 1.78831, + "98": 1.86819, + "99": 2.53966, + "100": 1.66196 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json index f8d630aa812..ec8b55de8b4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.96459, - "2": 10.95289, + "1": 10.96455, + "2": 10.95282, "3": 10.95803, - "4": 10.95244, - "5": 10.95248, - "6": 10.95358, - "7": 10.95137, - "8": 10.95114, - "9": 10.95832, - "10": 10.94456, - "11": 10.93658, - "12": 10.94322, - "13": 10.94355, - "14": 10.94301, - "15": 10.91488, - "16": 10.90027, - "17": 10.91041, - "18": 10.90618, - "19": 10.90182, - "20": 10.80739, + "4": 10.95242, + "5": 10.95253, + "6": 10.95361, + "7": 10.95139, + "8": 10.95109, + "9": 10.9583, + "10": 10.94445, + "11": 10.93655, + "12": 10.94323, + "13": 10.94349, + "14": 10.94298, + "15": 10.9148, + "16": 10.90029, + "17": 10.91038, + "18": 10.9061, + "19": 10.90183, + "20": 10.80731, "21": 10.79426, - "22": 10.80475, - "23": 10.78953, - "24": 10.77488, - "25": 10.76205, - "26": 10.75482, - "27": 10.71916, + "22": 10.80476, + "23": 10.78952, + "24": 10.7749, + "25": 10.76207, + "26": 10.75487, + "27": 10.7191, "28": 10.63825, - "29": 10.60841, - "30": 10.5862, - "31": 10.59086, + "29": 10.60844, + "30": 10.58628, + "31": 10.59085, "32": 10.57067, - "33": 10.53985, + "33": 10.5399, "34": 10.49922, "35": 10.50003, - "36": 10.48548, - "37": 10.44895, - "38": 10.45016, - "39": 10.41721, - "40": 10.39508, - "41": 10.37463, - "42": 10.36017, - "43": 10.33036, - "44": 10.31849, - "45": 10.31555, - "46": 10.27647, - "47": 10.26789, - "48": 10.21799, - "49": 10.2135, - "50": 10.22186 + "36": 10.48545, + "37": 10.449, + "38": 10.45019, + "39": 10.41717, + "40": 10.39513, + "41": 10.37477, + "42": 10.36024, + "43": 10.33042, + "44": 10.31854, + "45": 10.31567, + "46": 10.27655, + "47": 10.26794, + "48": 10.218, + "49": 10.21352, + "50": 10.22191 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1870.0, - "2": 1721.0, - "3": 1853.0, - "4": 1776.0, - "5": 1802.0, - "6": 1726.0, - "7": 1933.0, - "8": 1721.0, - "9": 1713.0, - "10": 1749.0, - "11": 1674.0, - "12": 1715.0, - "13": 1859.0, - "14": 1838.0, - "15": 1604.0, - "16": 1770.0, - "17": 1822.0, - "18": 1841.0, - "19": 1648.0, - "20": 1756.0, - "21": 1757.0, - "22": 1731.0, - "23": 1766.0, - "24": 1747.0, - "25": 1668.0, - "26": 1803.0, - "27": 1821.0, - "28": 1767.0, - "29": 1855.0, - "30": 1754.0, - "31": 1997.0, - "32": 1955.0, - "33": 1901.0, - "34": 1959.0, - "35": 2073.0, - "36": 1993.0, - "37": 2152.0, - "38": 2056.0, - "39": 2209.0, - "40": 2206.0, - "41": 2226.0, - "42": 2022.0, - "43": 2364.0, - "44": 2204.0, - "45": 2468.0, - "46": 2353.0, - "47": 2380.0, - "48": 2508.0, - "49": 2742.0, - "50": 2494.0 + "1": 1770.0, + "2": 1689.0, + "3": 1797.0, + "4": 1700.0, + "5": 1742.0, + "6": 1764.0, + "7": 1919.0, + "8": 1639.0, + "9": 1699.0, + "10": 1768.0, + "11": 1700.0, + "12": 1684.0, + "13": 1854.0, + "14": 1902.0, + "15": 1571.0, + "16": 1714.0, + "17": 1833.0, + "18": 1822.0, + "19": 1617.0, + "20": 1646.0, + "21": 1886.0, + "22": 1768.0, + "23": 1768.0, + "24": 1731.0, + "25": 1700.0, + "26": 1793.0, + "27": 1755.0, + "28": 1799.0, + "29": 1894.0, + "30": 1820.0, + "31": 2013.0, + "32": 1935.0, + "33": 1968.0, + "34": 1991.0, + "35": 2074.0, + "36": 1985.0, + "37": 2199.0, + "38": 2055.0, + "39": 2176.0, + "40": 2288.0, + "41": 2224.0, + "42": 2000.0, + "43": 2326.0, + "44": 2167.0, + "45": 2443.0, + "46": 2432.0, + "47": 2371.0, + "48": 2599.0, + "49": 2687.0, + "50": 2576.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.02254, - "3": 0.12256, - "4": 0.124, - "5": 0.12217, - "6": 0.12111, - "7": 0.12333, - "8": 0.12063, - "9": 0.12652, - "10": 0.12735, - "11": 0.12426, - "12": 0.12945, - "13": 0.12193, - "14": 0.11999, - "15": 0.11908, - "16": 0.11846, - "17": 0.1209, - "18": 0.12537, - "19": 0.12222, - "20": 0.12047, - "21": 0.12121, - "22": 0.12216, - "23": 0.12369, - "24": 0.12274, - "25": 0.1184, - "26": 0.1209, - "27": 0.12289, - "28": 0.12143, - "29": 0.12385, - "30": 0.12146, - "31": 0.12197, - "32": 0.12352, - "33": 0.12352, - "34": 0.12118, - "35": 0.12124, - "36": 0.12427, - "37": 0.12542, - "38": 0.1255, - "39": 0.12545, - "40": 0.11878, - "41": 0.12106, - "42": 0.12173, - "43": 0.12535, - "44": 0.12147, - "45": 0.12391, - "46": 0.12543, - "47": 0.12069, - "48": 0.12417, - "49": 0.12556, - "50": 0.12213 + "2": 6.80439, + "3": 0.12278, + "4": 0.12656, + "5": 0.12636, + "6": 0.12974, + "7": 0.12306, + "8": 0.13031, + "9": 0.12364, + "10": 0.12627, + "11": 0.12809, + "12": 0.12499, + "13": 0.12513, + "14": 0.12552, + "15": 0.12225, + "16": 0.12412, + "17": 0.12658, + "18": 0.12674, + "19": 0.12753, + "20": 0.12974, + "21": 0.12791, + "22": 0.1249, + "23": 0.12856, + "24": 0.12698, + "25": 0.13036, + "26": 0.12751, + "27": 0.13162, + "28": 0.12974, + "29": 0.12731, + "30": 0.13187, + "31": 0.12532, + "32": 0.12943, + "33": 0.12869, + "34": 0.12905, + "35": 0.13065, + "36": 0.13073, + "37": 0.13095, + "38": 0.13019, + "39": 0.12771, + "40": 0.12318, + "41": 0.12503, + "42": 0.12716, + "43": 0.12707, + "44": 0.13139, + "45": 0.13245, + "46": 0.1289, + "47": 0.12388, + "48": 0.12625, + "49": 0.12749, + "50": 0.12424 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json index cccd6bf832b..9dadd15e1d8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93805, - "2": 10.93722, - "3": 10.93622, + "1": 10.93802, + "2": 10.93731, + "3": 10.93625, "4": 10.94202, - "5": 10.93341, - "6": 10.93239, - "7": 10.9378, - "8": 10.93336, - "9": 10.93523, - "10": 10.93708, - "11": 10.92541, - "12": 10.92478, - "13": 10.92498, - "14": 10.913, - "15": 10.89303, - "16": 10.88848, - "17": 10.89973, - "18": 10.87744, - "19": 10.87629, - "20": 10.79078, - "21": 10.78028, - "22": 10.77488, - "23": 10.77493, + "5": 10.93343, + "6": 10.93236, + "7": 10.93773, + "8": 10.93332, + "9": 10.93526, + "10": 10.93703, + "11": 10.92535, + "12": 10.92481, + "13": 10.92504, + "14": 10.91293, + "15": 10.89304, + "16": 10.88857, + "17": 10.89967, + "18": 10.87741, + "19": 10.87633, + "20": 10.79086, + "21": 10.78034, + "22": 10.77493, + "23": 10.77501, "24": 10.7363, - "25": 10.74377, + "25": 10.74381, "26": 10.72424, - "27": 10.69604, - "28": 10.62337, - "29": 10.6032, - "30": 10.58016, - "31": 10.56927, - "32": 10.54831, - "33": 10.51986, - "34": 10.48554, - "35": 10.49212, - "36": 10.47099, - "37": 10.43652, - "38": 10.44075, - "39": 10.40244, - "40": 10.39244, - "41": 10.3684, - "42": 10.34588, - "43": 10.32635, - "44": 10.28947, - "45": 10.30802, - "46": 10.26791, - "47": 10.25248, - "48": 10.20359, - "49": 10.2011, - "50": 10.20928 + "27": 10.69605, + "28": 10.62339, + "29": 10.60317, + "30": 10.58018, + "31": 10.5693, + "32": 10.54838, + "33": 10.51995, + "34": 10.48559, + "35": 10.4921, + "36": 10.47113, + "37": 10.43649, + "38": 10.44083, + "39": 10.40248, + "40": 10.39252, + "41": 10.36846, + "42": 10.34595, + "43": 10.32632, + "44": 10.28954, + "45": 10.30813, + "46": 10.268, + "47": 10.25254, + "48": 10.2036, + "49": 10.20116, + "50": 10.2093 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1793.0, - "2": 1843.0, - "3": 1760.0, - "4": 1725.0, - "5": 1766.0, - "6": 1737.0, - "7": 2008.0, - "8": 1734.0, - "9": 1810.0, - "10": 1808.0, - "11": 1609.0, - "12": 1629.0, - "13": 1860.0, - "14": 1810.0, - "15": 1650.0, - "16": 1834.0, - "17": 1793.0, - "18": 1784.0, - "19": 1710.0, - "20": 1737.0, - "21": 1773.0, - "22": 1764.0, - "23": 1702.0, - "24": 1899.0, - "25": 1744.0, - "26": 1836.0, - "27": 1790.0, - "28": 1869.0, - "29": 1952.0, - "30": 1853.0, - "31": 1955.0, - "32": 1968.0, - "33": 2035.0, - "34": 2015.0, - "35": 2058.0, - "36": 1929.0, - "37": 2202.0, - "38": 2144.0, - "39": 2101.0, - "40": 2199.0, - "41": 2343.0, - "42": 1977.0, - "43": 2341.0, - "44": 2128.0, - "45": 2447.0, - "46": 2354.0, - "47": 2413.0, - "48": 2564.0, - "49": 2774.0, - "50": 2598.0 + "1": 1847.0, + "2": 1732.0, + "3": 1830.0, + "4": 1772.0, + "5": 1804.0, + "6": 1733.0, + "7": 1999.0, + "8": 1621.0, + "9": 1833.0, + "10": 1782.0, + "11": 1707.0, + "12": 1783.0, + "13": 1896.0, + "14": 1815.0, + "15": 1743.0, + "16": 1712.0, + "17": 1797.0, + "18": 1865.0, + "19": 1728.0, + "20": 1733.0, + "21": 1827.0, + "22": 1744.0, + "23": 1595.0, + "24": 1910.0, + "25": 1714.0, + "26": 1789.0, + "27": 1814.0, + "28": 1818.0, + "29": 1976.0, + "30": 1876.0, + "31": 2034.0, + "32": 2040.0, + "33": 2031.0, + "34": 2007.0, + "35": 2082.0, + "36": 2001.0, + "37": 2176.0, + "38": 2130.0, + "39": 2111.0, + "40": 2350.0, + "41": 2375.0, + "42": 2005.0, + "43": 2345.0, + "44": 2223.0, + "45": 2457.0, + "46": 2379.0, + "47": 2442.0, + "48": 2573.0, + "49": 2799.0, + "50": 2611.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.96224, - "3": 2.3218, - "4": 2.17825, - "5": 1.70713, - "6": 1.96774, - "7": 1.53587, - "8": 2.45493, - "9": 2.97904, - "10": 2.29226, - "11": 2.69633, - "12": 2.13665, - "13": 2.50903, - "14": 1.89119, - "15": 2.14381, - "16": 2.00981, - "17": 2.91275, - "18": 2.36625, - "19": 1.95486, - "20": 1.67776, - "21": 2.00545, - "22": 1.32206, - "23": 2.11073, - "24": 1.80439, - "25": 2.60464, - "26": 2.20412, - "27": 1.87553, - "28": 2.02727, - "29": 1.56077, - "30": 1.56044, - "31": 1.52813, - "32": 2.45511, - "33": 1.44263, - "34": 1.61757, - "35": 2.05391, - "36": 2.0777, - "37": 1.84558, - "38": 1.6175, - "39": 2.18046, - "40": 1.51716, - "41": 1.96475, - "42": 2.54763, - "43": 1.81373, - "44": 1.64848, - "45": 1.99012, - "46": 1.71886, - "47": 1.7574, - "48": 2.11413, - "49": 1.46823, - "50": 2.03845 + "2": 6.21606, + "3": 1.97608, + "4": 1.94061, + "5": 1.5354, + "6": 1.77599, + "7": 1.51187, + "8": 1.93925, + "9": 2.22402, + "10": 1.97714, + "11": 1.9106, + "12": 1.52091, + "13": 2.00601, + "14": 2.03699, + "15": 1.48235, + "16": 1.71005, + "17": 2.08309, + "18": 1.81162, + "19": 1.79643, + "20": 1.31336, + "21": 1.94426, + "22": 1.59852, + "23": 2.05692, + "24": 1.55973, + "25": 2.10097, + "26": 2.24769, + "27": 1.73432, + "28": 1.92005, + "29": 1.76013, + "30": 1.9476, + "31": 1.56786, + "32": 2.83037, + "33": 1.59895, + "34": 1.6255, + "35": 2.07613, + "36": 2.20641, + "37": 2.02294, + "38": 1.46356, + "39": 2.32875, + "40": 2.13036, + "41": 2.10688, + "42": 2.6333, + "43": 1.69339, + "44": 1.68345, + "45": 2.28812, + "46": 1.63661, + "47": 1.92765, + "48": 2.20627, + "49": 1.65288, + "50": 2.03483 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json index 456fd0e7b9b..86456790e63 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.96459, - "2": 10.95289, + "1": 10.96455, + "2": 10.95282, "3": 10.95803, - "4": 10.95244, - "5": 10.95248, - "6": 10.95358, - "7": 10.95137, - "8": 10.95114, - "9": 10.95832, - "10": 10.94456, - "11": 10.93658, - "12": 10.94324, - "13": 10.9435, - "14": 10.94301, - "15": 10.91487, - "16": 10.90024, - "17": 10.91038, - "18": 10.9062, + "4": 10.95242, + "5": 10.95253, + "6": 10.95361, + "7": 10.95139, + "8": 10.95109, + "9": 10.9583, + "10": 10.94445, + "11": 10.93655, + "12": 10.94322, + "13": 10.94347, + "14": 10.943, + "15": 10.91485, + "16": 10.90026, + "17": 10.91034, + "18": 10.90611, "19": 10.90187, - "20": 10.80737, - "21": 10.79428, - "22": 10.80475, - "23": 10.78952, - "24": 10.7749, - "25": 10.76208, - "26": 10.75479, + "20": 10.80734, + "21": 10.79429, + "22": 10.80478, + "23": 10.78955, + "24": 10.77489, + "25": 10.76205, + "26": 10.75484, "27": 10.71913, - "28": 10.63827, - "29": 10.60838, - "30": 10.58622, - "31": 10.59086, - "32": 10.57066, - "33": 10.53982, - "34": 10.4992, - "35": 10.50001, - "36": 10.48547, + "28": 10.63824, + "29": 10.60843, + "30": 10.58626, + "31": 10.59084, + "32": 10.5707, + "33": 10.53991, + "34": 10.49925, + "35": 10.50003, + "36": 10.48545, "37": 10.44898, - "38": 10.45018, - "39": 10.41717, - "40": 10.39509, - "41": 10.37465, - "42": 10.36016, - "43": 10.33036, - "44": 10.31849, - "45": 10.31556, - "46": 10.27648, - "47": 10.2679, - "48": 10.21803, - "49": 10.21349, - "50": 10.22186 + "38": 10.45024, + "39": 10.41718, + "40": 10.39512, + "41": 10.37475, + "42": 10.36025, + "43": 10.33043, + "44": 10.31856, + "45": 10.31564, + "46": 10.27655, + "47": 10.26796, + "48": 10.21798, + "49": 10.21355, + "50": 10.22191 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1870.0, - "2": 1721.0, - "3": 1853.0, - "4": 1776.0, - "5": 1802.0, - "6": 1726.0, - "7": 1933.0, - "8": 1749.0, - "9": 1799.0, - "10": 1776.0, - "11": 1689.0, - "12": 1714.0, - "13": 1782.0, - "14": 1911.0, - "15": 1584.0, - "16": 1800.0, - "17": 1882.0, - "18": 1841.0, - "19": 1641.0, - "20": 1762.0, - "21": 1886.0, - "22": 1745.0, - "23": 1791.0, - "24": 1806.0, - "25": 1719.0, - "26": 1725.0, - "27": 1790.0, - "28": 1672.0, - "29": 1907.0, - "30": 1768.0, - "31": 2048.0, - "32": 1910.0, - "33": 1954.0, - "34": 2036.0, - "35": 2084.0, - "36": 2015.0, - "37": 2210.0, - "38": 2054.0, - "39": 2199.0, - "40": 2183.0, - "41": 2318.0, - "42": 1917.0, - "43": 2402.0, - "44": 2091.0, - "45": 2433.0, - "46": 2384.0, - "47": 2340.0, - "48": 2594.0, - "49": 2773.0, - "50": 2561.0 + "1": 1770.0, + "2": 1689.0, + "3": 1797.0, + "4": 1700.0, + "5": 1742.0, + "6": 1764.0, + "7": 1919.0, + "8": 1639.0, + "9": 1699.0, + "10": 1768.0, + "11": 1719.0, + "12": 1710.0, + "13": 1860.0, + "14": 1850.0, + "15": 1697.0, + "16": 1768.0, + "17": 1884.0, + "18": 1779.0, + "19": 1665.0, + "20": 1773.0, + "21": 1693.0, + "22": 1695.0, + "23": 1735.0, + "24": 1775.0, + "25": 1597.0, + "26": 1762.0, + "27": 1849.0, + "28": 1723.0, + "29": 1983.0, + "30": 1775.0, + "31": 2077.0, + "32": 2029.0, + "33": 1981.0, + "34": 2022.0, + "35": 1970.0, + "36": 2026.0, + "37": 2204.0, + "38": 2088.0, + "39": 2220.0, + "40": 2225.0, + "41": 2278.0, + "42": 1977.0, + "43": 2394.0, + "44": 2109.0, + "45": 2455.0, + "46": 2344.0, + "47": 2417.0, + "48": 2602.0, + "49": 2782.0, + "50": 2527.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 552730112.0, - "2": 552730112.0, - "3": 552730112.0, - "4": 552730112.0, - "5": 552730112.0, - "6": 552730112.0, - "7": 552730112.0, - "8": 552730112.0, - "9": 552730112.0, - "10": 552730112.0, - "11": 552730112.0, - "12": 552730112.0, - "13": 552730112.0, - "14": 552730112.0, - "15": 552730112.0, - "16": 552730112.0, - "17": 552730112.0, - "18": 552730112.0, - "19": 552730112.0, - "20": 552730112.0, - "21": 552730112.0, - "22": 552730112.0, - "23": 552730112.0, - "24": 552730112.0, - "25": 552730112.0, - "26": 552730112.0, - "27": 552730112.0, - "28": 552730112.0, - "29": 552730112.0, - "30": 552730112.0, - "31": 552730112.0, - "32": 552730112.0, - "33": 552730112.0, - "34": 552730112.0, - "35": 552730112.0, - "36": 552730112.0, - "37": 552730112.0, - "38": 552730112.0, - "39": 552730112.0, - "40": 552730112.0, - "41": 552730112.0, - "42": 552730112.0, - "43": 552730112.0, - "44": 552730112.0, - "45": 552730112.0, - "46": 552730112.0, - "47": 552730112.0, - "48": 552730112.0, - "49": 552730112.0, - "50": 552730112.0 + "1": 552732160.0, + "2": 552732160.0, + "3": 552732160.0, + "4": 552732160.0, + "5": 552732160.0, + "6": 552732160.0, + "7": 552732160.0, + "8": 552732160.0, + "9": 552732160.0, + "10": 552732160.0, + "11": 552732160.0, + "12": 552732160.0, + "13": 552732160.0, + "14": 552732160.0, + "15": 552732160.0, + "16": 552732160.0, + "17": 552732160.0, + "18": 552732160.0, + "19": 552732160.0, + "20": 552732160.0, + "21": 552732160.0, + "22": 552732160.0, + "23": 552732160.0, + "24": 552732160.0, + "25": 552732160.0, + "26": 552732160.0, + "27": 552732160.0, + "28": 552732160.0, + "29": 552732160.0, + "30": 552732160.0, + "31": 552732160.0, + "32": 552732160.0, + "33": 552732160.0, + "34": 552732160.0, + "35": 552732160.0, + "36": 552732160.0, + "37": 552732160.0, + "38": 552732160.0, + "39": 552732160.0, + "40": 552732160.0, + "41": 552732160.0, + "42": 552732160.0, + "43": 552732160.0, + "44": 552732160.0, + "45": 552732160.0, + "46": 552732160.0, + "47": 552732160.0, + "48": 552732160.0, + "49": 552732160.0, + "50": 552732160.0 } }, "mem-max-allocated-bytes": { @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 3798208000.0, - "2": 3942621696.0, - "3": 3942621696.0, - "4": 3942621696.0, - "5": 3942621696.0, - "6": 3942621696.0, - "7": 3942621696.0, - "8": 3942621696.0, - "9": 3942621696.0, - "10": 3942621696.0, - "11": 3942621696.0, - "12": 3942621696.0, - "13": 3942621696.0, - "14": 3942621696.0, - "15": 3942621696.0, - "16": 3942621696.0, - "17": 3942621696.0, - "18": 3942621696.0, - "19": 3942621696.0, - "20": 3942621696.0, - "21": 3942621696.0, - "22": 3942621696.0, - "23": 3942621696.0, - "24": 3942621696.0, - "25": 3942621696.0, - "26": 3942621696.0, - "27": 3942621696.0, - "28": 3942621696.0, - "29": 3942621696.0, - "30": 3942621696.0, - "31": 3942621696.0, - "32": 3942621696.0, - "33": 3942621696.0, - "34": 3942621696.0, - "35": 3942621696.0, - "36": 3942621696.0, - "37": 3942621696.0, - "38": 3942621696.0, - "39": 3942621696.0, - "40": 3942621696.0, - "41": 3942621696.0, - "42": 3942621696.0, - "43": 3942621696.0, - "44": 3942621696.0, - "45": 3942621696.0, - "46": 3942621696.0, - "47": 3942621696.0, - "48": 3942621696.0, - "49": 3942621696.0, - "50": 3942621696.0 + "2": 3942493696.0, + "3": 3942493696.0, + "4": 3942493696.0, + "5": 3942493696.0, + "6": 3942493696.0, + "7": 3942493696.0, + "8": 3942493696.0, + "9": 3942493696.0, + "10": 3942493696.0, + "11": 3942493696.0, + "12": 3942493696.0, + "13": 3942493696.0, + "14": 3942493696.0, + "15": 3942493696.0, + "16": 3942493696.0, + "17": 3942493696.0, + "18": 3942493696.0, + "19": 3942493696.0, + "20": 3942493696.0, + "21": 3942493696.0, + "22": 3942493696.0, + "23": 3942493696.0, + "24": 3942493696.0, + "25": 3942493696.0, + "26": 3942493696.0, + "27": 3942493696.0, + "28": 3942493696.0, + "29": 3942493696.0, + "30": 3942493696.0, + "31": 3942493696.0, + "32": 3942493696.0, + "33": 3942493696.0, + "34": 3942493696.0, + "35": 3942493696.0, + "36": 3942493696.0, + "37": 3942493696.0, + "38": 3942493696.0, + "39": 3942493696.0, + "40": 3942493696.0, + "41": 3942493696.0, + "42": 3942493696.0, + "43": 3942493696.0, + "44": 3942493696.0, + "45": 3942493696.0, + "46": 3942493696.0, + "47": 3942493696.0, + "48": 3942493696.0, + "49": 3942493696.0, + "50": 3942493696.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.88607, - "3": 0.12398, - "4": 0.12548, - "5": 0.12607, - "6": 0.12929, - "7": 0.12376, - "8": 0.12665, - "9": 0.12501, - "10": 0.12664, - "11": 0.35519, - "12": 0.12845, - "13": 0.12535, - "14": 0.12538, - "15": 0.12546, - "16": 0.1268, - "17": 0.12657, - "18": 0.12789, - "19": 0.12916, - "20": 0.12759, - "21": 0.35106, - "22": 0.13041, - "23": 0.13084, - "24": 0.1295, - "25": 0.12814, - "26": 0.12883, - "27": 0.13337, - "28": 0.13301, - "29": 0.13306, - "30": 0.13191, - "31": 0.35306, - "32": 0.12533, - "33": 0.12783, - "34": 0.12786, - "35": 0.12871, - "36": 0.12987, - "37": 0.12593, - "38": 0.12987, - "39": 0.12794, - "40": 0.12959, - "41": 0.35629, - "42": 0.12969, - "43": 0.12633, - "44": 0.13152, - "45": 0.13006, - "46": 0.12691, - "47": 0.12701, - "48": 0.13036, - "49": 0.13103, - "50": 0.12901 + "2": 7.65464, + "3": 0.12922, + "4": 0.12653, + "5": 0.12758, + "6": 0.13014, + "7": 0.12594, + "8": 0.12537, + "9": 0.13285, + "10": 0.13091, + "11": 0.36187, + "12": 0.12712, + "13": 0.12522, + "14": 0.12289, + "15": 0.12392, + "16": 0.12773, + "17": 0.12794, + "18": 0.13017, + "19": 0.13078, + "20": 0.12791, + "21": 0.37168, + "22": 0.13484, + "23": 0.13862, + "24": 0.13601, + "25": 0.13472, + "26": 0.13561, + "27": 0.14146, + "28": 0.13937, + "29": 0.13795, + "30": 0.13956, + "31": 0.37218, + "32": 0.13758, + "33": 0.13871, + "34": 0.13539, + "35": 0.13707, + "36": 0.1416, + "37": 0.13882, + "38": 0.14068, + "39": 0.13979, + "40": 0.13496, + "41": 0.36437, + "42": 0.13247, + "43": 0.1334, + "44": 0.13221, + "45": 0.13068, + "46": 0.13453, + "47": 0.12944, + "48": 0.13269, + "49": 0.1376, + "50": 0.12857 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json index e3bb29cc9bb..2d87d29f1f7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.93805, - "2": 10.93722, - "3": 10.93622, + "1": 10.93802, + "2": 10.93731, + "3": 10.93625, "4": 10.94202, - "5": 10.93341, - "6": 10.93239, - "7": 10.9378, - "8": 10.93336, - "9": 10.93523, - "10": 10.93708, - "11": 10.92541, + "5": 10.93343, + "6": 10.93236, + "7": 10.93773, + "8": 10.93332, + "9": 10.93526, + "10": 10.93703, + "11": 10.92537, "12": 10.92478, - "13": 10.92498, - "14": 10.913, - "15": 10.89303, - "16": 10.88848, - "17": 10.89973, - "18": 10.87744, - "19": 10.8763, - "20": 10.79072, + "13": 10.925, + "14": 10.91289, + "15": 10.89306, + "16": 10.88857, + "17": 10.89968, + "18": 10.87739, + "19": 10.87631, + "20": 10.79086, "21": 10.78034, - "22": 10.77489, - "23": 10.77493, - "24": 10.73628, - "25": 10.74375, - "26": 10.72423, - "27": 10.69602, - "28": 10.62335, - "29": 10.60318, - "30": 10.58016, - "31": 10.56934, - "32": 10.54832, - "33": 10.51985, - "34": 10.48553, - "35": 10.4921, - "36": 10.47101, - "37": 10.43653, - "38": 10.44075, - "39": 10.40247, - "40": 10.39246, - "41": 10.36838, - "42": 10.34589, - "43": 10.32638, - "44": 10.28949, - "45": 10.30803, - "46": 10.26792, - "47": 10.25248, - "48": 10.20356, - "49": 10.20108, - "50": 10.20926 + "22": 10.77492, + "23": 10.77501, + "24": 10.73633, + "25": 10.74379, + "26": 10.72426, + "27": 10.69605, + "28": 10.62339, + "29": 10.60324, + "30": 10.58015, + "31": 10.56932, + "32": 10.5484, + "33": 10.51996, + "34": 10.48559, + "35": 10.49212, + "36": 10.47112, + "37": 10.43649, + "38": 10.44078, + "39": 10.40249, + "40": 10.3925, + "41": 10.36846, + "42": 10.34595, + "43": 10.32635, + "44": 10.28952, + "45": 10.30812, + "46": 10.26798, + "47": 10.25255, + "48": 10.20361, + "49": 10.20112, + "50": 10.20931 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1793.0, - "2": 1843.0, - "3": 1760.0, - "4": 1725.0, - "5": 1766.0, - "6": 1737.0, - "7": 2008.0, - "8": 1734.0, - "9": 1810.0, - "10": 1808.0, - "11": 1609.0, - "12": 1629.0, - "13": 1860.0, - "14": 1810.0, - "15": 1650.0, - "16": 1834.0, - "17": 1793.0, - "18": 1785.0, - "19": 1678.0, - "20": 1648.0, - "21": 1762.0, - "22": 1718.0, - "23": 1666.0, - "24": 1852.0, - "25": 1701.0, - "26": 1848.0, - "27": 1834.0, - "28": 1871.0, - "29": 2032.0, - "30": 1892.0, - "31": 2038.0, - "32": 1994.0, - "33": 2040.0, - "34": 2067.0, - "35": 2022.0, - "36": 1876.0, + "1": 1847.0, + "2": 1732.0, + "3": 1830.0, + "4": 1772.0, + "5": 1804.0, + "6": 1733.0, + "7": 1999.0, + "8": 1621.0, + "9": 1833.0, + "10": 1782.0, + "11": 1669.0, + "12": 1672.0, + "13": 1869.0, + "14": 1823.0, + "15": 1676.0, + "16": 1761.0, + "17": 1797.0, + "18": 1883.0, + "19": 1643.0, + "20": 1736.0, + "21": 1802.0, + "22": 1713.0, + "23": 1674.0, + "24": 1885.0, + "25": 1673.0, + "26": 1912.0, + "27": 1877.0, + "28": 1921.0, + "29": 1967.0, + "30": 1857.0, + "31": 1996.0, + "32": 2028.0, + "33": 1988.0, + "34": 2080.0, + "35": 2049.0, + "36": 1936.0, "37": 2201.0, - "38": 2209.0, - "39": 2128.0, - "40": 2321.0, - "41": 2376.0, - "42": 1900.0, - "43": 2332.0, - "44": 2199.0, - "45": 2583.0, - "46": 2490.0, - "47": 2430.0, - "48": 2588.0, - "49": 2814.0, - "50": 2547.0 + "38": 2121.0, + "39": 2141.0, + "40": 2336.0, + "41": 2327.0, + "42": 1891.0, + "43": 2274.0, + "44": 2249.0, + "45": 2399.0, + "46": 2389.0, + "47": 2451.0, + "48": 2595.0, + "49": 2728.0, + "50": 2500.0 } }, "mem-allocated-bytes": { @@ -150,7 +150,7 @@ "30": 761634304.0, "31": 761634304.0, "32": 761634304.0, - "33": 762158592.0, + "33": 761634304.0, "34": 761634304.0, "35": 761634304.0, "36": 761634304.0, @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 3868768768.0, - "2": 4153623552.0, - "3": 4153623552.0, - "4": 4153623552.0, - "5": 4153623552.0, - "6": 4153623552.0, - "7": 4153623552.0, - "8": 4153623552.0, - "9": 4153623552.0, - "10": 4153623552.0, - "11": 4153623552.0, - "12": 4153623552.0, - "13": 4153623552.0, - "14": 4153623552.0, - "15": 4153623552.0, - "16": 4153623552.0, - "17": 4153623552.0, - "18": 4153623552.0, - "19": 4153623552.0, - "20": 4153623552.0, - "21": 4153623552.0, - "22": 4153623552.0, - "23": 4153623552.0, - "24": 4153623552.0, - "25": 4153623552.0, - "26": 4153623552.0, - "27": 4153623552.0, - "28": 4153623552.0, - "29": 4153623552.0, - "30": 4153623552.0, - "31": 4153623552.0, - "32": 4153623552.0, - "33": 4153623552.0, - "34": 4153623552.0, - "35": 4153623552.0, - "36": 4153623552.0, - "37": 4153623552.0, - "38": 4153623552.0, - "39": 4153623552.0, - "40": 4153623552.0, - "41": 4153623552.0, - "42": 4153623552.0, - "43": 4153623552.0, - "44": 4153623552.0, - "45": 4153623552.0, - "46": 4153623552.0, - "47": 4153623552.0, - "48": 4153623552.0, - "49": 4153623552.0, - "50": 4153623552.0 + "2": 4152574464.0, + "3": 4152576000.0, + "4": 4152576000.0, + "5": 4153099264.0, + "6": 4153099264.0, + "7": 4153099264.0, + "8": 4153099264.0, + "9": 4153099264.0, + "10": 4153099264.0, + "11": 4153099264.0, + "12": 4153099264.0, + "13": 4153099264.0, + "14": 4153099264.0, + "15": 4153099264.0, + "16": 4153099264.0, + "17": 4153099264.0, + "18": 4153099264.0, + "19": 4153099264.0, + "20": 4153099264.0, + "21": 4153099264.0, + "22": 4153099264.0, + "23": 4153099264.0, + "24": 4153099264.0, + "25": 4153099264.0, + "26": 4153099264.0, + "27": 4153099264.0, + "28": 4153099264.0, + "29": 4153099264.0, + "30": 4153099264.0, + "31": 4153099264.0, + "32": 4153099264.0, + "33": 4153099264.0, + "34": 4153100800.0, + "35": 4153100800.0, + "36": 4153100800.0, + "37": 4153100800.0, + "38": 4153100800.0, + "39": 4153100800.0, + "40": 4153100800.0, + "41": 4153100800.0, + "42": 4153100800.0, + "43": 4153100800.0, + "44": 4153100800.0, + "45": 4153100800.0, + "46": 4153100800.0, + "47": 4153100800.0, + "48": 4153100800.0, + "49": 4153100800.0, + "50": 4153100800.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.67523, - "3": 2.27552, - "4": 1.79312, - "5": 1.72626, - "6": 1.68961, - "7": 1.90367, - "8": 2.16647, - "9": 1.81026, - "10": 2.21507, - "11": 2.34321, - "12": 1.7015, - "13": 1.82959, - "14": 2.34278, - "15": 1.58959, - "16": 1.801, - "17": 2.13046, - "18": 1.78159, - "19": 2.03672, - "20": 1.64137, - "21": 1.97082, - "22": 1.6638, - "23": 2.23761, - "24": 1.80264, - "25": 2.31038, - "26": 2.42274, - "27": 1.82868, - "28": 1.91514, - "29": 1.58088, - "30": 1.9589, - "31": 1.75576, - "32": 3.03192, - "33": 1.51195, - "34": 1.58649, - "35": 2.12525, - "36": 2.22916, - "37": 2.02127, - "38": 1.51169, - "39": 2.3559, - "40": 2.17475, - "41": 2.11859, - "42": 2.73378, - "43": 1.74168, - "44": 1.8561, - "45": 2.44526, - "46": 1.83278, - "47": 1.88328, - "48": 2.2646, - "49": 1.79655, - "50": 2.10757 + "2": 7.32594, + "3": 2.15625, + "4": 2.14749, + "5": 1.72561, + "6": 2.00805, + "7": 1.46708, + "8": 2.00565, + "9": 2.58429, + "10": 1.91496, + "11": 1.90743, + "12": 1.55845, + "13": 1.68991, + "14": 1.92551, + "15": 1.39011, + "16": 1.7573, + "17": 2.17998, + "18": 1.43005, + "19": 1.68691, + "20": 1.65547, + "21": 1.65516, + "22": 1.44177, + "23": 1.72687, + "24": 1.57953, + "25": 2.17134, + "26": 2.87971, + "27": 2.28146, + "28": 2.11008, + "29": 1.66826, + "30": 1.74137, + "31": 1.28767, + "32": 2.44945, + "33": 2.00977, + "34": 2.08021, + "35": 2.27409, + "36": 2.73021, + "37": 2.07222, + "38": 1.87297, + "39": 2.34834, + "40": 2.36872, + "41": 1.99132, + "42": 2.83266, + "43": 2.06779, + "44": 1.97666, + "45": 2.32735, + "46": 1.38703, + "47": 1.88381, + "48": 2.37769, + "49": 1.73053, + "50": 2.48462 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json index f6aa3b73aee..6125df21c98 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95856, + "1": 10.96462, + "2": 10.95232, + "3": 10.95858, "4": 10.95193, - "5": 10.95264, - "6": 10.95378, + "5": 10.95261, + "6": 10.95383, "7": 10.95159, - "8": 10.95135, - "9": 10.95869, - "10": 10.94454, - "11": 10.93683, - "12": 10.94341, - "13": 10.94309, - "14": 10.94286, - "15": 10.91495, - "16": 10.90027, - "17": 10.91108, - "18": 10.90589, - "19": 10.90126, - "20": 10.80771, - "21": 10.79443, - "22": 10.80462, - "23": 10.78963, - "24": 10.77416, - "25": 10.7606, - "26": 10.75391, - "27": 10.7187, - "28": 10.63859, - "29": 10.60793, - "30": 10.58595, - "31": 10.59034, - "32": 10.56919, - "33": 10.53723, + "8": 10.95138, + "9": 10.95865, + "10": 10.94448, + "11": 10.93684, + "12": 10.94335, + "13": 10.94307, + "14": 10.94284, + "15": 10.91488, + "16": 10.90025, + "17": 10.911, + "18": 10.90585, + "19": 10.90125, + "20": 10.80769, + "21": 10.7944, + "22": 10.80464, + "23": 10.78956, + "24": 10.77419, + "25": 10.76059, + "26": 10.75389, + "27": 10.71867, + "28": 10.63858, + "29": 10.60799, + "30": 10.58598, + "31": 10.59032, + "32": 10.56926, + "33": 10.5373, "34": 10.49524, - "35": 10.49635, - "36": 10.48327, - "37": 10.44677, - "38": 10.4482, + "35": 10.49637, + "36": 10.48329, + "37": 10.44678, + "38": 10.44825, "39": 10.41533, - "40": 10.39307, - "41": 10.37226, - "42": 10.35597, + "40": 10.3931, + "41": 10.37237, + "42": 10.35606, "43": 10.32385, - "44": 10.31083, - "45": 10.30676, - "46": 10.266, - "47": 10.2582, - "48": 10.20966, - "49": 10.20575, - "50": 10.21353, - "51": 10.21324, - "52": 10.15841, - "53": 10.16271, - "54": 10.12809, + "44": 10.31086, + "45": 10.30682, + "46": 10.26607, + "47": 10.25827, + "48": 10.20968, + "49": 10.20578, + "50": 10.21359, + "51": 10.21329, + "52": 10.15843, + "53": 10.16273, + "54": 10.12814, "55": 10.09863, - "56": 10.12108, - "57": 10.10886, - "58": 10.11352, - "59": 10.05657, - "60": 10.07841, - "61": 10.03375, - "62": 10.00109, - "63": 10.06759, - "64": 10.01686, + "56": 10.12107, + "57": 10.10884, + "58": 10.11353, + "59": 10.0566, + "60": 10.07848, + "61": 10.03382, + "62": 10.00115, + "63": 10.06764, + "64": 10.0169, "65": 9.98864, - "66": 10.02938, - "67": 9.99989, - "68": 9.96572, - "69": 9.97412, - "70": 9.95973, - "71": 9.98131, - "72": 9.96658, - "73": 9.94918, - "74": 9.94054, - "75": 9.91022, - "76": 9.93992, - "77": 9.93684, + "66": 10.02946, + "67": 9.99988, + "68": 9.96578, + "69": 9.97419, + "70": 9.95978, + "71": 9.98134, + "72": 9.96659, + "73": 9.94919, + "74": 9.94059, + "75": 9.91026, + "76": 9.93997, + "77": 9.93687, "78": 9.88872, - "79": 9.8896, - "80": 9.90383, - "81": 9.92953, - "82": 9.86789, - "83": 9.83326, - "84": 9.77449, - "85": 9.75504, - "86": 9.8559, - "87": 9.87809, - "88": 9.84953, - "89": 9.78708, - "90": 9.78521, - "91": 9.78362, - "92": 9.77827, - "93": 9.70912, - "94": 9.78142, - "95": 9.76786, - "96": 9.75262, - "97": 9.7003, - "98": 9.71823, + "79": 9.88964, + "80": 9.90392, + "81": 9.92963, + "82": 9.86795, + "83": 9.8333, + "84": 9.77456, + "85": 9.75509, + "86": 9.85593, + "87": 9.87807, + "88": 9.84954, + "89": 9.78714, + "90": 9.78526, + "91": 9.78366, + "92": 9.77828, + "93": 9.70916, + "94": 9.78145, + "95": 9.76788, + "96": 9.75266, + "97": 9.70036, + "98": 9.7183, "99": 9.77348, - "100": 9.6534 + "100": 9.65348 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1840.0, - "4": 1776.0, - "5": 1738.0, - "6": 1754.0, - "7": 1820.0, - "8": 1756.0, - "9": 1687.0, - "10": 1731.0, - "11": 1692.0, - "12": 1629.0, - "13": 1860.0, - "14": 1863.0, - "15": 1580.0, - "16": 1765.0, - "17": 1790.0, - "18": 1790.0, - "19": 1722.0, - "20": 1727.0, - "21": 1833.0, - "22": 1757.0, - "23": 1805.0, - "24": 1881.0, - "25": 1782.0, - "26": 1795.0, - "27": 1810.0, - "28": 1781.0, - "29": 1842.0, - "30": 1817.0, - "31": 1973.0, - "32": 1968.0, - "33": 1939.0, - "34": 1961.0, - "35": 2034.0, - "36": 2031.0, - "37": 2203.0, - "38": 2042.0, - "39": 2123.0, - "40": 2164.0, - "41": 2272.0, - "42": 1959.0, - "43": 2318.0, - "44": 2152.0, - "45": 2439.0, - "46": 2332.0, - "47": 2369.0, - "48": 2574.0, - "49": 2872.0, - "50": 2550.0, - "51": 2403.0, - "52": 2649.0, - "53": 2555.0, - "54": 2693.0, - "55": 2363.0, - "56": 2705.0, - "57": 2210.0, - "58": 3467.0, - "59": 2889.0, - "60": 2925.0, - "61": 2651.0, - "62": 3162.0, - "63": 3222.0, - "64": 3604.0, - "65": 2772.0, - "66": 2971.0, - "67": 3800.0, - "68": 3304.0, - "69": 2983.0, - "70": 3327.0, - "71": 3168.0, - "72": 2835.0, - "73": 3418.0, - "74": 3295.0, - "75": 3027.0, - "76": 3280.0, - "77": 3707.0, - "78": 3342.0, - "79": 3320.0, - "80": 3060.0, - "81": 3370.0, - "82": 3036.0, - "83": 3024.0, - "84": 2865.0, - "85": 2793.0, - "86": 3030.0, - "87": 2852.0, - "88": 3000.0, - "89": 3197.0, - "90": 3708.0, - "91": 2908.0, - "92": 3069.0, - "93": 3103.0, - "94": 2906.0, - "95": 3462.0, - "96": 3323.0, - "97": 3535.0, - "98": 3390.0, - "99": 3077.0, - "100": 3122.0 + "1": 1759.0, + "2": 1644.0, + "3": 1707.0, + "4": 1731.0, + "5": 1784.0, + "6": 1746.0, + "7": 1886.0, + "8": 1676.0, + "9": 1742.0, + "10": 1790.0, + "11": 1710.0, + "12": 1691.0, + "13": 1829.0, + "14": 1930.0, + "15": 1544.0, + "16": 1792.0, + "17": 1770.0, + "18": 1843.0, + "19": 1713.0, + "20": 1685.0, + "21": 1765.0, + "22": 1728.0, + "23": 1696.0, + "24": 1902.0, + "25": 1684.0, + "26": 1774.0, + "27": 1732.0, + "28": 1812.0, + "29": 1968.0, + "30": 1819.0, + "31": 1993.0, + "32": 1940.0, + "33": 1927.0, + "34": 1973.0, + "35": 2130.0, + "36": 1984.0, + "37": 2200.0, + "38": 2051.0, + "39": 2217.0, + "40": 2276.0, + "41": 2308.0, + "42": 2065.0, + "43": 2317.0, + "44": 2204.0, + "45": 2395.0, + "46": 2409.0, + "47": 2404.0, + "48": 2601.0, + "49": 2895.0, + "50": 2543.0, + "51": 2440.0, + "52": 2670.0, + "53": 2604.0, + "54": 2598.0, + "55": 2549.0, + "56": 2611.0, + "57": 2194.0, + "58": 3601.0, + "59": 2800.0, + "60": 2976.0, + "61": 2615.0, + "62": 3120.0, + "63": 3048.0, + "64": 3516.0, + "65": 2729.0, + "66": 3001.0, + "67": 3661.0, + "68": 3390.0, + "69": 2960.0, + "70": 3335.0, + "71": 3221.0, + "72": 3007.0, + "73": 3363.0, + "74": 3229.0, + "75": 3100.0, + "76": 3237.0, + "77": 3528.0, + "78": 3363.0, + "79": 3337.0, + "80": 2968.0, + "81": 3560.0, + "82": 3006.0, + "83": 3132.0, + "84": 2916.0, + "85": 2723.0, + "86": 3066.0, + "87": 2765.0, + "88": 3111.0, + "89": 3277.0, + "90": 3660.0, + "91": 2861.0, + "92": 3046.0, + "93": 3143.0, + "94": 2927.0, + "95": 3412.0, + "96": 3335.0, + "97": 3499.0, + "98": 3440.0, + "99": 2979.0, + "100": 3130.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.76577, - "3": 0.10888, - "4": 0.10801, - "5": 0.10507, - "6": 0.10756, - "7": 0.10683, - "8": 0.11011, - "9": 0.10535, - "10": 0.10554, - "11": 0.10849, - "12": 0.107, - "13": 0.10477, - "14": 0.10681, - "15": 0.10597, - "16": 0.10862, - "17": 0.10543, - "18": 0.11036, - "19": 0.10405, - "20": 0.1044, - "21": 0.10336, - "22": 0.10377, - "23": 0.1077, - "24": 0.10989, - "25": 0.10804, - "26": 0.10778, - "27": 0.11124, - "28": 0.10764, - "29": 0.11152, - "30": 0.10649, - "31": 0.11059, - "32": 0.10649, - "33": 0.10428, - "34": 0.1065, - "35": 0.10718, - "36": 0.11006, - "37": 0.11092, - "38": 0.10975, - "39": 0.10918, - "40": 0.10574, - "41": 0.108, - "42": 0.10816, - "43": 0.10971, - "44": 0.10792, - "45": 0.10927, - "46": 0.10431, - "47": 0.10912, - "48": 0.10941, - "49": 0.10696, - "50": 0.10682, - "51": 0.16269, - "52": 0.10898, - "53": 0.10613, - "54": 0.10907, - "55": 0.1061, - "56": 0.10726, - "57": 0.10671, - "58": 0.10813, - "59": 0.11101, - "60": 0.10731, - "61": 0.11075, - "62": 0.11072, - "63": 0.10816, - "64": 0.10653, - "65": 0.10565, - "66": 0.1087, - "67": 0.10738, - "68": 0.11142, - "69": 0.10798, - "70": 0.1069, - "71": 0.10471, - "72": 0.10769, - "73": 0.1079, - "74": 0.10753, - "75": 0.10839, - "76": 0.10612, - "77": 0.10624, - "78": 0.10666, - "79": 0.10679, - "80": 0.10858, - "81": 0.10683, - "82": 0.10569, - "83": 0.1092, - "84": 0.1069, - "85": 0.10972, - "86": 0.10149, - "87": 0.10613, - "88": 0.1073, - "89": 0.10598, - "90": 0.10894, - "91": 0.10465, - "92": 0.10771, - "93": 0.10564, - "94": 0.10242, - "95": 0.1048, - "96": 0.1089, - "97": 0.10597, - "98": 0.10765, - "99": 0.1079, - "100": 0.10669 + "2": 7.35085, + "3": 0.11684, + "4": 0.11786, + "5": 0.11583, + "6": 0.1194, + "7": 0.11608, + "8": 0.11543, + "9": 0.11471, + "10": 0.11676, + "11": 0.11988, + "12": 0.11758, + "13": 0.11913, + "14": 0.11447, + "15": 0.1163, + "16": 0.11612, + "17": 0.11747, + "18": 0.11519, + "19": 0.11744, + "20": 0.11571, + "21": 0.12473, + "22": 0.11815, + "23": 0.12167, + "24": 0.11856, + "25": 0.11714, + "26": 0.11696, + "27": 0.12074, + "28": 0.12112, + "29": 0.12304, + "30": 0.12099, + "31": 0.11897, + "32": 0.11775, + "33": 0.119, + "34": 0.11239, + "35": 0.11494, + "36": 0.11497, + "37": 0.11558, + "38": 0.11809, + "39": 0.11721, + "40": 0.11436, + "41": 0.11639, + "42": 0.11688, + "43": 0.11561, + "44": 0.11944, + "45": 0.12281, + "46": 0.11409, + "47": 0.11427, + "48": 0.11709, + "49": 0.12313, + "50": 0.11721, + "51": 0.16891, + "52": 0.11291, + "53": 0.11243, + "54": 0.11403, + "55": 0.11525, + "56": 0.1155, + "57": 0.11666, + "58": 0.11164, + "59": 0.11665, + "60": 0.11227, + "61": 0.11766, + "62": 0.11568, + "63": 0.11698, + "64": 0.11304, + "65": 0.11542, + "66": 0.11282, + "67": 0.11338, + "68": 0.11454, + "69": 0.11399, + "70": 0.11533, + "71": 0.11498, + "72": 0.1165, + "73": 0.12067, + "74": 0.1164, + "75": 0.11916, + "76": 0.11779, + "77": 0.11611, + "78": 0.11454, + "79": 0.11698, + "80": 0.11874, + "81": 0.11243, + "82": 0.11611, + "83": 0.1156, + "84": 0.1137, + "85": 0.11721, + "86": 0.11353, + "87": 0.11667, + "88": 0.12184, + "89": 0.11348, + "90": 0.11968, + "91": 0.1182, + "92": 0.11588, + "93": 0.11772, + "94": 0.11446, + "95": 0.11744, + "96": 0.12047, + "97": 0.11532, + "98": 0.11621, + "99": 0.1163, + "100": 0.11484 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json index cb3e6b857a1..fe8a91022b3 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.94232, - "5": 10.93349, - "6": 10.93269, - "7": 10.93796, - "8": 10.93356, - "9": 10.93555, - "10": 10.93692, - "11": 10.92585, - "12": 10.92412, - "13": 10.92504, - "14": 10.91248, - "15": 10.89335, - "16": 10.88815, - "17": 10.89968, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.9423, + "5": 10.93351, + "6": 10.93267, + "7": 10.93789, + "8": 10.93352, + "9": 10.93554, + "10": 10.93686, + "11": 10.92579, + "12": 10.9241, + "13": 10.92503, + "14": 10.91244, + "15": 10.89334, + "16": 10.88825, + "17": 10.89966, "18": 10.87789, - "19": 10.87549, - "20": 10.79108, - "21": 10.78001, - "22": 10.77459, - "23": 10.77485, - "24": 10.73559, - "25": 10.74282, - "26": 10.72347, - "27": 10.6955, - "28": 10.62303, - "29": 10.60325, - "30": 10.57991, - "31": 10.56891, - "32": 10.54647, - "33": 10.51723, - "34": 10.48161, - "35": 10.48816, - "36": 10.46791, - "37": 10.43373, - "38": 10.43848, - "39": 10.40028, - "40": 10.3903, - "41": 10.36588, - "42": 10.34114, - "43": 10.3191, - "44": 10.28097, - "45": 10.2984, - "46": 10.25723, - "47": 10.24193, - "48": 10.19501, - "49": 10.19305, - "50": 10.20072, - "51": 10.19623, - "52": 10.14789, - "53": 10.15718, - "54": 10.12042, - "55": 10.09009, + "19": 10.87552, + "20": 10.79109, + "21": 10.78002, + "22": 10.77465, + "23": 10.77492, + "24": 10.73562, + "25": 10.7428, + "26": 10.72344, + "27": 10.69551, + "28": 10.62308, + "29": 10.60327, + "30": 10.57993, + "31": 10.56896, + "32": 10.54649, + "33": 10.51737, + "34": 10.48165, + "35": 10.48815, + "36": 10.468, + "37": 10.43369, + "38": 10.4385, + "39": 10.40031, + "40": 10.39035, + "41": 10.36597, + "42": 10.34125, + "43": 10.31908, + "44": 10.281, + "45": 10.29848, + "46": 10.25735, + "47": 10.24198, + "48": 10.19505, + "49": 10.19309, + "50": 10.20075, + "51": 10.19627, + "52": 10.14797, + "53": 10.1572, + "54": 10.12045, + "55": 10.09012, "56": 10.1185, - "57": 10.10154, - "58": 10.11659, - "59": 10.06156, - "60": 10.07213, - "61": 10.02683, - "62": 9.9928, - "63": 10.06735, + "57": 10.10153, + "58": 10.11664, + "59": 10.06158, + "60": 10.07214, + "61": 10.02685, + "62": 9.99289, + "63": 10.0674, "64": 10.02093, - "65": 9.99324, - "66": 10.0227, - "67": 9.98726, - "68": 9.95238, - "69": 9.97227, - "70": 9.95945, - "71": 9.97771, - "72": 9.94822, - "73": 9.94072, - "74": 9.9307, - "75": 9.89817, - "76": 9.9397, - "77": 9.9293, - "78": 9.87561, - "79": 9.88327, - "80": 9.89532, - "81": 9.91711, - "82": 9.85934, - "83": 9.81914, - "84": 9.75546, - "85": 9.74598, - "86": 9.847, - "87": 9.87922, - "88": 9.85037, - "89": 9.78552, - "90": 9.77222, - "91": 9.78185, - "92": 9.76913, - "93": 9.70722, - "94": 9.77765, - "95": 9.77216, - "96": 9.75877, - "97": 9.69519, - "98": 9.72098, - "99": 9.76711, - "100": 9.65463 + "65": 9.99329, + "66": 10.02279, + "67": 9.98731, + "68": 9.95243, + "69": 9.97239, + "70": 9.95946, + "71": 9.97778, + "72": 9.9483, + "73": 9.9408, + "74": 9.93077, + "75": 9.89819, + "76": 9.93977, + "77": 9.92936, + "78": 9.87568, + "79": 9.8833, + "80": 9.89538, + "81": 9.91715, + "82": 9.85935, + "83": 9.81917, + "84": 9.75554, + "85": 9.74603, + "86": 9.84705, + "87": 9.87929, + "88": 9.85046, + "89": 9.78555, + "90": 9.7723, + "91": 9.78186, + "92": 9.76916, + "93": 9.70728, + "94": 9.77774, + "95": 9.77221, + "96": 9.75888, + "97": 9.69527, + "98": 9.72103, + "99": 9.76715, + "100": 9.65471 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1823.0, - "4": 1751.0, - "5": 1765.0, - "6": 1747.0, + "1": 1791.0, + "2": 1794.0, + "3": 1792.0, + "4": 1665.0, + "5": 1760.0, + "6": 1738.0, "7": 2058.0, - "8": 1609.0, - "9": 1791.0, - "10": 1746.0, - "11": 1671.0, - "12": 1691.0, - "13": 1884.0, - "14": 1910.0, - "15": 1701.0, - "16": 1714.0, - "17": 1896.0, - "18": 1865.0, - "19": 1790.0, - "20": 1786.0, - "21": 1826.0, - "22": 1753.0, - "23": 1655.0, - "24": 1889.0, - "25": 1720.0, - "26": 1854.0, - "27": 1720.0, - "28": 1872.0, - "29": 1931.0, - "30": 1873.0, - "31": 2004.0, - "32": 1975.0, - "33": 1992.0, - "34": 2001.0, - "35": 2033.0, - "36": 1907.0, - "37": 2178.0, - "38": 2172.0, - "39": 2174.0, - "40": 2324.0, - "41": 2413.0, - "42": 1986.0, - "43": 2354.0, - "44": 2233.0, - "45": 2482.0, - "46": 2411.0, - "47": 2288.0, + "8": 1660.0, + "9": 1819.0, + "10": 1668.0, + "11": 1678.0, + "12": 1671.0, + "13": 1904.0, + "14": 1855.0, + "15": 1633.0, + "16": 1676.0, + "17": 1850.0, + "18": 1852.0, + "19": 1680.0, + "20": 1701.0, + "21": 1721.0, + "22": 1703.0, + "23": 1684.0, + "24": 1886.0, + "25": 1803.0, + "26": 1906.0, + "27": 1789.0, + "28": 1775.0, + "29": 1957.0, + "30": 1897.0, + "31": 1968.0, + "32": 1987.0, + "33": 2015.0, + "34": 1994.0, + "35": 2010.0, + "36": 1915.0, + "37": 2261.0, + "38": 2056.0, + "39": 2112.0, + "40": 2336.0, + "41": 2314.0, + "42": 2051.0, + "43": 2405.0, + "44": 2206.0, + "45": 2437.0, + "46": 2410.0, + "47": 2351.0, "48": 2670.0, - "49": 2785.0, - "50": 2757.0, - "51": 2528.0, - "52": 2712.0, - "53": 2618.0, - "54": 2827.0, - "55": 2526.0, - "56": 2672.0, - "57": 2172.0, - "58": 3466.0, - "59": 2770.0, - "60": 2893.0, - "61": 2653.0, - "62": 3098.0, - "63": 3284.0, - "64": 3400.0, - "65": 2751.0, - "66": 3072.0, - "67": 3801.0, - "68": 3389.0, - "69": 2897.0, - "70": 3096.0, - "71": 3056.0, - "72": 2962.0, - "73": 3260.0, - "74": 3118.0, - "75": 3090.0, - "76": 3252.0, - "77": 3597.0, - "78": 3083.0, - "79": 3233.0, - "80": 2958.0, - "81": 3295.0, - "82": 3257.0, - "83": 3143.0, - "84": 3047.0, - "85": 2752.0, - "86": 3251.0, - "87": 2866.0, - "88": 3149.0, - "89": 2803.0, - "90": 3609.0, - "91": 3035.0, - "92": 2997.0, - "93": 3106.0, - "94": 3118.0, - "95": 3399.0, - "96": 3534.0, - "97": 3611.0, - "98": 3729.0, - "99": 3121.0, - "100": 3262.0 + "49": 2792.0, + "50": 2550.0, + "51": 2466.0, + "52": 2738.0, + "53": 2570.0, + "54": 2791.0, + "55": 2465.0, + "56": 2782.0, + "57": 2212.0, + "58": 3482.0, + "59": 2828.0, + "60": 2865.0, + "61": 2695.0, + "62": 3143.0, + "63": 3205.0, + "64": 3346.0, + "65": 2665.0, + "66": 3013.0, + "67": 3770.0, + "68": 3440.0, + "69": 2919.0, + "70": 3019.0, + "71": 3032.0, + "72": 2969.0, + "73": 3343.0, + "74": 3143.0, + "75": 3120.0, + "76": 3156.0, + "77": 3601.0, + "78": 3094.0, + "79": 3297.0, + "80": 3066.0, + "81": 3228.0, + "82": 3232.0, + "83": 3248.0, + "84": 3002.0, + "85": 2885.0, + "86": 3133.0, + "87": 2915.0, + "88": 3126.0, + "89": 2864.0, + "90": 3532.0, + "91": 2981.0, + "92": 2952.0, + "93": 3250.0, + "94": 3233.0, + "95": 3500.0, + "96": 3551.0, + "97": 3640.0, + "98": 3738.0, + "99": 3123.0, + "100": 3338.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.86698, - "3": 2.23709, - "4": 1.93789, - "5": 1.54945, - "6": 1.8486, - "7": 1.56946, - "8": 1.90325, - "9": 2.25159, - "10": 1.61854, - "11": 1.86873, - "12": 1.51085, - "13": 1.65829, - "14": 1.50138, - "15": 1.37256, - "16": 1.6651, - "17": 2.38708, - "18": 1.47357, - "19": 1.73005, - "20": 1.71666, - "21": 1.85755, - "22": 1.201, - "23": 1.92128, - "24": 1.29664, - "25": 2.19689, - "26": 2.1285, - "27": 1.55775, - "28": 1.70244, - "29": 1.58319, - "30": 1.54706, - "31": 1.26425, - "32": 1.94276, - "33": 1.38332, - "34": 1.46561, - "35": 1.98009, - "36": 1.949, - "37": 1.6834, - "38": 1.32733, - "39": 2.09247, - "40": 1.55836, - "41": 1.76807, - "42": 2.34285, - "43": 1.58718, - "44": 1.63324, - "45": 1.93623, - "46": 1.62651, - "47": 1.86707, - "48": 1.87844, - "49": 1.24183, - "50": 1.7234, - "51": 1.58703, - "52": 1.75205, - "53": 1.63938, - "54": 1.82167, - "55": 1.76283, - "56": 2.35548, - "57": 1.26384, - "58": 2.15886, - "59": 1.68252, - "60": 1.35111, - "61": 2.03462, - "62": 2.04452, - "63": 1.48275, - "64": 1.97048, - "65": 1.96045, - "66": 2.08338, - "67": 1.70836, - "68": 1.37627, - "69": 1.7677, - "70": 1.41652, - "71": 1.93616, - "72": 1.35719, - "73": 1.77487, - "74": 1.82048, - "75": 1.66406, - "76": 1.75487, - "77": 1.82277, - "78": 1.68683, - "79": 2.15529, - "80": 1.71822, - "81": 2.0064, - "82": 1.90058, - "83": 1.70485, - "84": 1.45853, - "85": 1.63833, - "86": 1.66802, - "87": 1.95107, - "88": 1.92734, - "89": 1.7966, - "90": 1.59094, - "91": 1.74259, - "92": 2.19268, - "93": 1.33438, - "94": 1.98423, - "95": 1.84764, - "96": 2.10523, - "97": 1.76641, - "98": 1.78903, - "99": 2.35571, - "100": 1.59126 + "2": 6.27939, + "3": 1.52013, + "4": 1.52963, + "5": 1.25298, + "6": 1.62045, + "7": 1.30624, + "8": 2.05277, + "9": 1.79727, + "10": 1.45516, + "11": 1.48103, + "12": 1.18067, + "13": 1.28398, + "14": 1.57994, + "15": 1.25265, + "16": 1.55898, + "17": 2.09894, + "18": 1.24844, + "19": 1.54495, + "20": 1.33131, + "21": 1.74081, + "22": 1.15877, + "23": 1.58071, + "24": 1.44285, + "25": 1.97494, + "26": 1.8774, + "27": 1.3787, + "28": 1.4607, + "29": 1.21091, + "30": 1.7324, + "31": 1.30675, + "32": 1.89656, + "33": 1.23259, + "34": 1.23731, + "35": 1.81938, + "36": 1.49916, + "37": 1.41475, + "38": 1.32079, + "39": 1.79984, + "40": 1.44552, + "41": 1.44147, + "42": 2.01487, + "43": 1.50688, + "44": 1.26353, + "45": 1.88968, + "46": 1.29531, + "47": 1.30061, + "48": 1.52518, + "49": 1.293, + "50": 1.55596, + "51": 1.26493, + "52": 1.72755, + "53": 1.34715, + "54": 1.62856, + "55": 1.45182, + "56": 1.98597, + "57": 1.04511, + "58": 1.61898, + "59": 1.4393, + "60": 1.45383, + "61": 1.78155, + "62": 1.83684, + "63": 1.25657, + "64": 1.48816, + "65": 1.6864, + "66": 2.18367, + "67": 1.54212, + "68": 1.42666, + "69": 1.91323, + "70": 1.44951, + "71": 2.40885, + "72": 1.62463, + "73": 1.63097, + "74": 1.5043, + "75": 1.56858, + "76": 1.88887, + "77": 1.56466, + "78": 1.45973, + "79": 2.44976, + "80": 1.67073, + "81": 1.92434, + "82": 1.77207, + "83": 1.60468, + "84": 1.42906, + "85": 1.78511, + "86": 1.36402, + "87": 2.1063, + "88": 1.92258, + "89": 1.75017, + "90": 1.73529, + "91": 1.49172, + "92": 2.3693, + "93": 1.32151, + "94": 1.7964, + "95": 1.97499, + "96": 1.90961, + "97": 1.46316, + "98": 1.90676, + "99": 2.07742, + "100": 1.8251 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json index 8040aa62289..92c7e48e0da 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95854, - "4": 10.95197, - "5": 10.95261, + "1": 10.96462, + "2": 10.95232, + "3": 10.95859, + "4": 10.95192, + "5": 10.95262, "6": 10.95382, - "7": 10.95159, - "8": 10.95132, - "9": 10.9587, - "10": 10.94456, - "11": 10.93683, - "12": 10.94343, - "13": 10.94314, - "14": 10.94294, - "15": 10.91497, - "16": 10.90029, - "17": 10.9111, - "18": 10.90623, - "19": 10.90165, - "20": 10.80789, - "21": 10.79458, - "22": 10.80475, - "23": 10.78977, - "24": 10.77497, - "25": 10.76167, + "7": 10.95158, + "8": 10.95135, + "9": 10.95869, + "10": 10.9445, + "11": 10.93682, + "12": 10.94337, + "13": 10.94313, + "14": 10.94293, + "15": 10.91492, + "16": 10.90026, + "17": 10.91109, + "18": 10.90612, + "19": 10.90162, + "20": 10.80786, + "21": 10.79455, + "22": 10.80479, + "23": 10.78975, + "24": 10.77506, + "25": 10.76169, "26": 10.75516, - "27": 10.71956, - "28": 10.63913, - "29": 10.60851, - "30": 10.58649, - "31": 10.59096, - "32": 10.57108, - "33": 10.54031, - "34": 10.49895, - "35": 10.50028, - "36": 10.48631, - "37": 10.44931, - "38": 10.4507, - "39": 10.41767, - "40": 10.39538, - "41": 10.37482, - "42": 10.3606, - "43": 10.33045, - "44": 10.31901, - "45": 10.31602, - "46": 10.27652, - "47": 10.26823, - "48": 10.21834, - "49": 10.21388, - "50": 10.22196, - "51": 10.22216, - "52": 10.16751, - "53": 10.17223, - "54": 10.13795, - "55": 10.10895, + "27": 10.71953, + "28": 10.63907, + "29": 10.60855, + "30": 10.58653, + "31": 10.59094, + "32": 10.57111, + "33": 10.54035, + "34": 10.49898, + "35": 10.50036, + "36": 10.4863, + "37": 10.44938, + "38": 10.45068, + "39": 10.41773, + "40": 10.39546, + "41": 10.37489, + "42": 10.36063, + "43": 10.33047, + "44": 10.31911, + "45": 10.31612, + "46": 10.27656, + "47": 10.2683, + "48": 10.21835, + "49": 10.21389, + "50": 10.22203, + "51": 10.22221, + "52": 10.16757, + "53": 10.17229, + "54": 10.138, + "55": 10.10898, "56": 10.13223, - "57": 10.1214, + "57": 10.12137, "58": 10.12635, - "59": 10.07039, - "60": 10.09306, - "61": 10.04907, + "59": 10.07045, + "60": 10.0931, + "61": 10.04912, "62": 10.01778, - "63": 10.08391, - "64": 10.03345, - "65": 10.00557, - "66": 10.04739, - "67": 10.01996, - "68": 9.98697, - "69": 9.99635, - "70": 9.98279, - "71": 10.00545, - "72": 9.99249, - "73": 9.97618, - "74": 9.96756, - "75": 9.93703, - "76": 9.96635, - "77": 9.96325, - "78": 9.91485, - "79": 9.91547, - "80": 9.92964, - "81": 9.9558, - "82": 9.89447, - "83": 9.85962, - "84": 9.80306, - "85": 9.78431, - "86": 9.88636, - "87": 9.90914, - "88": 9.88157, - "89": 9.82173, - "90": 9.82085, - "91": 9.82114, - "92": 9.81671, - "93": 9.74868, + "63": 10.08401, + "64": 10.03352, + "65": 10.00559, + "66": 10.04745, + "67": 10.02001, + "68": 9.98705, + "69": 9.99636, + "70": 9.98282, + "71": 10.0055, + "72": 9.99252, + "73": 9.97621, + "74": 9.9676, + "75": 9.93709, + "76": 9.96638, + "77": 9.96329, + "78": 9.91487, + "79": 9.9155, + "80": 9.92968, + "81": 9.95591, + "82": 9.89454, + "83": 9.85968, + "84": 9.8031, + "85": 9.78436, + "86": 9.88639, + "87": 9.90911, + "88": 9.8816, + "89": 9.8218, + "90": 9.82094, + "91": 9.82118, + "92": 9.81674, + "93": 9.7487, "94": 9.82364, - "95": 9.811, - "96": 9.79716, - "97": 9.74559, - "98": 9.7656, - "99": 9.82203, - "100": 9.70436 + "95": 9.81102, + "96": 9.7972, + "97": 9.74564, + "98": 9.76566, + "99": 9.82205, + "100": 9.70444 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1787.0, - "4": 1675.0, - "5": 1820.0, - "6": 1678.0, - "7": 1861.0, - "8": 1749.0, - "9": 1745.0, - "10": 1815.0, - "11": 1687.0, - "12": 1746.0, - "13": 1767.0, - "14": 1933.0, - "15": 1608.0, - "16": 1731.0, - "17": 1843.0, - "18": 1811.0, - "19": 1698.0, - "20": 1761.0, - "21": 1798.0, - "22": 1807.0, - "23": 1700.0, - "24": 1838.0, - "25": 1714.0, - "26": 1779.0, - "27": 1769.0, - "28": 1784.0, - "29": 1850.0, - "30": 1800.0, - "31": 2098.0, - "32": 1919.0, - "33": 1958.0, - "34": 1939.0, - "35": 2002.0, - "36": 1970.0, - "37": 2256.0, - "38": 2144.0, - "39": 2190.0, - "40": 2230.0, - "41": 2287.0, - "42": 2018.0, - "43": 2414.0, - "44": 2132.0, - "45": 2497.0, - "46": 2470.0, - "47": 2384.0, - "48": 2553.0, - "49": 2756.0, - "50": 2473.0, - "51": 2360.0, - "52": 2657.0, - "53": 2568.0, - "54": 2665.0, - "55": 2431.0, - "56": 2636.0, - "57": 2269.0, - "58": 3524.0, - "59": 2901.0, - "60": 2907.0, - "61": 2656.0, - "62": 3131.0, - "63": 3144.0, - "64": 3566.0, - "65": 2764.0, - "66": 3027.0, - "67": 3535.0, - "68": 3272.0, - "69": 2912.0, - "70": 3362.0, - "71": 3120.0, - "72": 2865.0, - "73": 3395.0, - "74": 3241.0, - "75": 3118.0, - "76": 3261.0, - "77": 3678.0, - "78": 3193.0, - "79": 3321.0, - "80": 3038.0, - "81": 3427.0, - "82": 2976.0, - "83": 3100.0, - "84": 2986.0, - "85": 2684.0, - "86": 3033.0, - "87": 2888.0, - "88": 3016.0, - "89": 3192.0, - "90": 3783.0, - "91": 3055.0, - "92": 3053.0, - "93": 3151.0, - "94": 2886.0, - "95": 3384.0, - "96": 3437.0, - "97": 3594.0, - "98": 3294.0, - "99": 3050.0, - "100": 3147.0 + "1": 1759.0, + "2": 1644.0, + "3": 1736.0, + "4": 1783.0, + "5": 1782.0, + "6": 1759.0, + "7": 1975.0, + "8": 1703.0, + "9": 1743.0, + "10": 1740.0, + "11": 1686.0, + "12": 1707.0, + "13": 1844.0, + "14": 1792.0, + "15": 1617.0, + "16": 1755.0, + "17": 1772.0, + "18": 1850.0, + "19": 1711.0, + "20": 1735.0, + "21": 1832.0, + "22": 1710.0, + "23": 1745.0, + "24": 1863.0, + "25": 1685.0, + "26": 1782.0, + "27": 1728.0, + "28": 1716.0, + "29": 1901.0, + "30": 1867.0, + "31": 2036.0, + "32": 2025.0, + "33": 1906.0, + "34": 1928.0, + "35": 2040.0, + "36": 1924.0, + "37": 2258.0, + "38": 2111.0, + "39": 2127.0, + "40": 2234.0, + "41": 2314.0, + "42": 1988.0, + "43": 2306.0, + "44": 2077.0, + "45": 2420.0, + "46": 2388.0, + "47": 2346.0, + "48": 2653.0, + "49": 2773.0, + "50": 2409.0, + "51": 2356.0, + "52": 2637.0, + "53": 2611.0, + "54": 2635.0, + "55": 2533.0, + "56": 2604.0, + "57": 2235.0, + "58": 3558.0, + "59": 2899.0, + "60": 2953.0, + "61": 2699.0, + "62": 3095.0, + "63": 3265.0, + "64": 3535.0, + "65": 2783.0, + "66": 3039.0, + "67": 3734.0, + "68": 3381.0, + "69": 2857.0, + "70": 3468.0, + "71": 3162.0, + "72": 2982.0, + "73": 3439.0, + "74": 3386.0, + "75": 3136.0, + "76": 3295.0, + "77": 3595.0, + "78": 3263.0, + "79": 3243.0, + "80": 3081.0, + "81": 3378.0, + "82": 2984.0, + "83": 2960.0, + "84": 2966.0, + "85": 2778.0, + "86": 3096.0, + "87": 2789.0, + "88": 3109.0, + "89": 3222.0, + "90": 3747.0, + "91": 2964.0, + "92": 3082.0, + "93": 3211.0, + "94": 2881.0, + "95": 3408.0, + "96": 3439.0, + "97": 3412.0, + "98": 3441.0, + "99": 3046.0, + "100": 3117.0 } }, "mem-allocated-bytes": { @@ -273,7 +273,7 @@ "53": 766367232.0, "54": 766367232.0, "55": 766367232.0, - "56": 766367232.0, + "56": 767415808.0, "57": 766367232.0, "58": 766367232.0, "59": 766367232.0, @@ -378,53 +378,53 @@ "51": 2647494656.0, "52": 2647494656.0, "53": 2647494656.0, - "54": 2647494656.0, - "55": 2647494656.0, - "56": 2647494656.0, - "57": 2647494656.0, - "58": 2647494656.0, - "59": 2647494656.0, - "60": 2647494656.0, - "61": 2647494656.0, - "62": 2647494656.0, - "63": 2647494656.0, - "64": 2647494656.0, - "65": 2647494656.0, - "66": 2647494656.0, - "67": 2647494656.0, - "68": 2647494656.0, - "69": 2647494656.0, - "70": 2647494656.0, - "71": 2647494656.0, - "72": 2647494656.0, - "73": 2647494656.0, - "74": 2647494656.0, - "75": 2647494656.0, - "76": 2647494656.0, - "77": 2647494656.0, - "78": 2647494656.0, - "79": 2647494656.0, - "80": 2647494656.0, - "81": 2647494656.0, - "82": 2647494656.0, - "83": 2647494656.0, - "84": 2647494656.0, - "85": 2647494656.0, - "86": 2647494656.0, - "87": 2647494656.0, - "88": 2647494656.0, - "89": 2647494656.0, - "90": 2647494656.0, - "91": 2647494656.0, - "92": 2647494656.0, - "93": 2647494656.0, - "94": 2647494656.0, - "95": 2647494656.0, - "96": 2647494656.0, - "97": 2647494656.0, - "98": 2647494656.0, - "99": 2647494656.0, - "100": 2647494656.0 + "54": 2648411648.0, + "55": 2648411648.0, + "56": 2648411648.0, + "57": 2648411648.0, + "58": 2648411648.0, + "59": 2648411648.0, + "60": 2648411648.0, + "61": 2648411648.0, + "62": 2648411648.0, + "63": 2648411648.0, + "64": 2648411648.0, + "65": 2648411648.0, + "66": 2648411648.0, + "67": 2648411648.0, + "68": 2648411648.0, + "69": 2648411648.0, + "70": 2648411648.0, + "71": 2648411648.0, + "72": 2648411648.0, + "73": 2648411648.0, + "74": 2648411648.0, + "75": 2648411648.0, + "76": 2648411648.0, + "77": 2648411648.0, + "78": 2648411648.0, + "79": 2648411648.0, + "80": 2648411648.0, + "81": 2648411648.0, + "82": 2648411648.0, + "83": 2648411648.0, + "84": 2648411648.0, + "85": 2648411648.0, + "86": 2648411648.0, + "87": 2648411648.0, + "88": 2648411648.0, + "89": 2648411648.0, + "90": 2648411648.0, + "91": 2648411648.0, + "92": 2648411648.0, + "93": 2648411648.0, + "94": 2648411648.0, + "95": 2648411648.0, + "96": 2648411648.0, + "97": 2648411648.0, + "98": 2648411648.0, + "99": 2648411648.0, + "100": 2648411648.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.07645, - "3": 0.1094, - "4": 0.10987, - "5": 0.10883, - "6": 0.11143, - "7": 0.11116, - "8": 0.10899, - "9": 0.10822, - "10": 0.10745, - "11": 0.11042, - "12": 0.11135, - "13": 0.1125, - "14": 0.10741, - "15": 0.10975, - "16": 0.11197, - "17": 0.10864, - "18": 0.11007, - "19": 0.10982, - "20": 0.1076, - "21": 0.10764, - "22": 0.10691, - "23": 0.10504, - "24": 0.10802, - "25": 0.1067, - "26": 0.10698, - "27": 0.1106, - "28": 0.10955, - "29": 0.10759, - "30": 0.11102, - "31": 0.10646, - "32": 0.1044, - "33": 0.10659, - "34": 0.10918, - "35": 0.10885, - "36": 0.10532, - "37": 0.11209, - "38": 0.1102, - "39": 0.10672, - "40": 0.10605, - "41": 0.106, - "42": 0.10788, - "43": 0.1076, - "44": 0.1109, - "45": 0.10762, - "46": 0.1075, - "47": 0.10575, - "48": 0.10826, - "49": 0.1088, - "50": 0.10673, - "51": 0.14127, - "52": 0.12881, - "53": 0.10771, - "54": 0.11168, - "55": 0.10677, - "56": 0.10627, - "57": 0.10897, - "58": 0.1109, - "59": 0.115, - "60": 0.10854, - "61": 0.11013, - "62": 0.11119, - "63": 0.11036, - "64": 0.10763, - "65": 0.11107, - "66": 0.10841, - "67": 0.10371, - "68": 0.10647, - "69": 0.10758, - "70": 0.10966, - "71": 0.10927, - "72": 0.10884, - "73": 0.10782, - "74": 0.10843, - "75": 0.1068, - "76": 0.10932, - "77": 0.1077, - "78": 0.11001, - "79": 0.10886, - "80": 0.10761, - "81": 0.10634, - "82": 0.10517, - "83": 0.11168, - "84": 0.10628, - "85": 0.11065, - "86": 0.10353, - "87": 0.10602, - "88": 0.11046, - "89": 0.10846, - "90": 0.10902, - "91": 0.10778, - "92": 0.10978, - "93": 0.10922, - "94": 0.10581, - "95": 0.10786, - "96": 0.10787, - "97": 0.10592, - "98": 0.10606, - "99": 0.10961, - "100": 0.10901 + "2": 7.75851, + "3": 0.12089, + "4": 0.11876, + "5": 0.11595, + "6": 0.12081, + "7": 0.11528, + "8": 0.11709, + "9": 0.11722, + "10": 0.11786, + "11": 0.12448, + "12": 0.12018, + "13": 0.11968, + "14": 0.11843, + "15": 0.12008, + "16": 0.11821, + "17": 0.11689, + "18": 0.12068, + "19": 0.11906, + "20": 0.11608, + "21": 0.11792, + "22": 0.1176, + "23": 0.11691, + "24": 0.11556, + "25": 0.11468, + "26": 0.11387, + "27": 0.11529, + "28": 0.11342, + "29": 0.11332, + "30": 0.11042, + "31": 0.1139, + "32": 0.11298, + "33": 0.11504, + "34": 0.11215, + "35": 0.11361, + "36": 0.11537, + "37": 0.1153, + "38": 0.11487, + "39": 0.11548, + "40": 0.11085, + "41": 0.11401, + "42": 0.1119, + "43": 0.11713, + "44": 0.11537, + "45": 0.11464, + "46": 0.11686, + "47": 0.11085, + "48": 0.12265, + "49": 0.11648, + "50": 0.11992, + "51": 0.16821, + "52": 0.14539, + "53": 0.11919, + "54": 0.11826, + "55": 0.11396, + "56": 0.11758, + "57": 0.11475, + "58": 0.11494, + "59": 0.12039, + "60": 0.11269, + "61": 0.11646, + "62": 0.11784, + "63": 0.1169, + "64": 0.11512, + "65": 0.11282, + "66": 0.11947, + "67": 0.11248, + "68": 0.11576, + "69": 0.11224, + "70": 0.11253, + "71": 0.11525, + "72": 0.11655, + "73": 0.11681, + "74": 0.11957, + "75": 0.11977, + "76": 0.11523, + "77": 0.11872, + "78": 0.11786, + "79": 0.1197, + "80": 0.11733, + "81": 0.11545, + "82": 0.11681, + "83": 0.11813, + "84": 0.11388, + "85": 0.11598, + "86": 0.11231, + "87": 0.11432, + "88": 0.11844, + "89": 0.11483, + "90": 0.11671, + "91": 0.11284, + "92": 0.1148, + "93": 0.11619, + "94": 0.11324, + "95": 0.11572, + "96": 0.118, + "97": 0.11693, + "98": 0.11478, + "99": 0.11299, + "100": 0.11547 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json index 68a163b9bd4..123cba325e3 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.9423, - "5": 10.93352, - "6": 10.93273, - "7": 10.93793, - "8": 10.93357, - "9": 10.93552, - "10": 10.93693, - "11": 10.92585, - "12": 10.92413, - "13": 10.9251, - "14": 10.91258, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.94226, + "5": 10.93351, + "6": 10.93269, + "7": 10.93786, + "8": 10.93354, + "9": 10.93555, + "10": 10.93689, + "11": 10.92582, + "12": 10.92411, + "13": 10.92513, + "14": 10.91252, "15": 10.89335, - "16": 10.88822, - "17": 10.89973, - "18": 10.87821, + "16": 10.8883, + "17": 10.89968, + "18": 10.87816, "19": 10.87589, - "20": 10.79119, + "20": 10.79127, "21": 10.78019, - "22": 10.77474, - "23": 10.77501, - "24": 10.7364, - "25": 10.74383, - "26": 10.72466, - "27": 10.69626, - "28": 10.62353, - "29": 10.60379, - "30": 10.58048, + "22": 10.77477, + "23": 10.77506, + "24": 10.73639, + "25": 10.74386, + "26": 10.72465, + "27": 10.69633, + "28": 10.62354, + "29": 10.60385, + "30": 10.58047, "31": 10.56959, "32": 10.5486, - "33": 10.52046, - "34": 10.48553, - "35": 10.4923, - "36": 10.47086, - "37": 10.43635, - "38": 10.44109, - "39": 10.40281, - "40": 10.3928, - "41": 10.36862, - "42": 10.34609, - "43": 10.32634, - "44": 10.28964, - "45": 10.3083, - "46": 10.2682, - "47": 10.2525, - "48": 10.20372, - "49": 10.20119, - "50": 10.20922, - "51": 10.20528, - "52": 10.15731, + "33": 10.52062, + "34": 10.48556, + "35": 10.49231, + "36": 10.47101, + "37": 10.43633, + "38": 10.44118, + "39": 10.40284, + "40": 10.39291, + "41": 10.36868, + "42": 10.34616, + "43": 10.32631, + "44": 10.28969, + "45": 10.30834, + "46": 10.26826, + "47": 10.25255, + "48": 10.20375, + "49": 10.20125, + "50": 10.20932, + "51": 10.20534, + "52": 10.15739, "53": 10.16687, - "54": 10.13043, - "55": 10.10057, + "54": 10.1305, + "55": 10.10061, "56": 10.13013, - "57": 10.11459, - "58": 10.13004, - "59": 10.07594, - "60": 10.08697, - "61": 10.04225, - "62": 10.00953, - "63": 10.08368, - "64": 10.03719, - "65": 10.00999, - "66": 10.04021, - "67": 10.00686, - "68": 9.97346, - "69": 9.99438, - "70": 9.98228, - "71": 10.00183, - "72": 9.97365, - "73": 9.96698, - "74": 9.95724, - "75": 9.92479, - "76": 9.96623, + "57": 10.11465, + "58": 10.13007, + "59": 10.07595, + "60": 10.08702, + "61": 10.0423, + "62": 10.00957, + "63": 10.08371, + "64": 10.03718, + "65": 10.01, + "66": 10.04025, + "67": 10.00692, + "68": 9.97351, + "69": 9.99448, + "70": 9.98227, + "71": 10.00189, + "72": 9.97374, + "73": 9.96701, + "74": 9.95731, + "75": 9.92482, + "76": 9.96625, "77": 9.95576, - "78": 9.90172, - "79": 9.90899, - "80": 9.92107, - "81": 9.94317, + "78": 9.90177, + "79": 9.90906, + "80": 9.92113, + "81": 9.94323, "82": 9.88583, - "83": 9.84493, - "84": 9.7831, - "85": 9.77443, - "86": 9.87683, - "87": 9.91001, - "88": 9.882, + "83": 9.845, + "84": 9.78317, + "85": 9.77446, + "86": 9.87691, + "87": 9.91006, + "88": 9.88207, "89": 9.81985, - "90": 9.80796, - "91": 9.81902, - "92": 9.80779, - "93": 9.74751, - "94": 9.82029, - "95": 9.8157, - "96": 9.80356, - "97": 9.7406, - "98": 9.76894, - "99": 9.81594, - "100": 9.70555 + "90": 9.80799, + "91": 9.81907, + "92": 9.80784, + "93": 9.74759, + "94": 9.82035, + "95": 9.81578, + "96": 9.80362, + "97": 9.74067, + "98": 9.76896, + "99": 9.81595, + "100": 9.7056 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1721.0, - "4": 1751.0, - "5": 1756.0, - "6": 1766.0, - "7": 2096.0, - "8": 1769.0, - "9": 1799.0, - "10": 1745.0, - "11": 1650.0, - "12": 1774.0, - "13": 1839.0, - "14": 1831.0, - "15": 1653.0, - "16": 1694.0, - "17": 1777.0, - "18": 1814.0, - "19": 1660.0, - "20": 1774.0, - "21": 1735.0, - "22": 1698.0, - "23": 1720.0, - "24": 1857.0, - "25": 1696.0, - "26": 1895.0, - "27": 1727.0, - "28": 1790.0, - "29": 1978.0, - "30": 1883.0, - "31": 2011.0, - "32": 2009.0, - "33": 2065.0, - "34": 2005.0, - "35": 1998.0, - "36": 1949.0, - "37": 2233.0, - "38": 2134.0, - "39": 2029.0, - "40": 2320.0, - "41": 2310.0, - "42": 2012.0, - "43": 2229.0, - "44": 2253.0, - "45": 2455.0, - "46": 2413.0, - "47": 2499.0, - "48": 2654.0, - "49": 2695.0, - "50": 2556.0, - "51": 2497.0, - "52": 2697.0, + "1": 1791.0, + "2": 1794.0, + "3": 1774.0, + "4": 1707.0, + "5": 1776.0, + "6": 1774.0, + "7": 2058.0, + "8": 1681.0, + "9": 1838.0, + "10": 1766.0, + "11": 1778.0, + "12": 1661.0, + "13": 1915.0, + "14": 1858.0, + "15": 1693.0, + "16": 1659.0, + "17": 1816.0, + "18": 1875.0, + "19": 1664.0, + "20": 1712.0, + "21": 1799.0, + "22": 1673.0, + "23": 1716.0, + "24": 1890.0, + "25": 1735.0, + "26": 1871.0, + "27": 1782.0, + "28": 1841.0, + "29": 1861.0, + "30": 1950.0, + "31": 1975.0, + "32": 1915.0, + "33": 1943.0, + "34": 1992.0, + "35": 2041.0, + "36": 1911.0, + "37": 2255.0, + "38": 2085.0, + "39": 2135.0, + "40": 2250.0, + "41": 2304.0, + "42": 1952.0, + "43": 2355.0, + "44": 2183.0, + "45": 2496.0, + "46": 2410.0, + "47": 2384.0, + "48": 2587.0, + "49": 2671.0, + "50": 2538.0, + "51": 2466.0, + "52": 2723.0, "53": 2566.0, - "54": 2737.0, - "55": 2430.0, - "56": 2681.0, - "57": 2235.0, - "58": 3500.0, - "59": 2810.0, - "60": 2847.0, - "61": 2634.0, - "62": 3234.0, - "63": 3180.0, - "64": 3344.0, - "65": 2659.0, - "66": 2923.0, - "67": 3727.0, - "68": 3419.0, - "69": 2980.0, - "70": 3155.0, - "71": 3102.0, - "72": 2985.0, - "73": 3265.0, - "74": 3183.0, - "75": 3040.0, - "76": 3010.0, - "77": 3807.0, - "78": 3133.0, - "79": 3258.0, - "80": 3010.0, - "81": 3341.0, - "82": 3261.0, - "83": 3212.0, - "84": 2927.0, - "85": 2838.0, - "86": 3179.0, - "87": 2957.0, - "88": 3042.0, - "89": 2842.0, - "90": 3453.0, - "91": 3015.0, - "92": 3062.0, - "93": 3085.0, - "94": 3098.0, - "95": 3397.0, - "96": 3546.0, - "97": 3660.0, - "98": 3603.0, - "99": 3169.0, - "100": 3271.0 + "54": 2789.0, + "55": 2483.0, + "56": 2606.0, + "57": 2206.0, + "58": 3501.0, + "59": 2982.0, + "60": 2905.0, + "61": 2699.0, + "62": 3139.0, + "63": 3239.0, + "64": 3365.0, + "65": 2646.0, + "66": 3002.0, + "67": 3724.0, + "68": 3388.0, + "69": 2901.0, + "70": 3363.0, + "71": 3001.0, + "72": 2955.0, + "73": 3332.0, + "74": 3180.0, + "75": 3066.0, + "76": 3097.0, + "77": 3692.0, + "78": 3079.0, + "79": 3113.0, + "80": 3031.0, + "81": 3333.0, + "82": 3182.0, + "83": 3173.0, + "84": 2896.0, + "85": 2796.0, + "86": 3091.0, + "87": 2911.0, + "88": 3087.0, + "89": 2758.0, + "90": 3507.0, + "91": 3127.0, + "92": 2971.0, + "93": 3166.0, + "94": 3124.0, + "95": 3452.0, + "96": 3566.0, + "97": 3541.0, + "98": 3564.0, + "99": 3267.0, + "100": 3287.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 764269056.0, - "2": 764269056.0, - "3": 764269056.0, - "4": 764269056.0, - "5": 764269056.0, - "6": 764269056.0, - "7": 764269056.0, - "8": 764269056.0, - "9": 764269056.0, - "10": 764269056.0, - "11": 764269056.0, - "12": 764269056.0, - "13": 764269056.0, - "14": 764269056.0, - "15": 764269056.0, - "16": 764269056.0, - "17": 764269056.0, - "18": 764269056.0, - "19": 764269056.0, - "20": 764269056.0, - "21": 764269056.0, - "22": 764269056.0, - "23": 764269056.0, - "24": 764269056.0, - "25": 764269056.0, - "26": 764269056.0, - "27": 764269056.0, - "28": 764269056.0, - "29": 764269056.0, - "30": 764269056.0, - "31": 764269056.0, - "32": 764269056.0, - "33": 764269056.0, - "34": 764269056.0, - "35": 764269056.0, - "36": 764269056.0, - "37": 764269056.0, - "38": 764269056.0, - "39": 764269056.0, - "40": 764269056.0, - "41": 764269056.0, - "42": 764269056.0, - "43": 764269056.0, - "44": 764269056.0, - "45": 764269056.0, - "46": 764269056.0, - "47": 764269056.0, - "48": 764269056.0, - "49": 764269056.0, - "50": 764269056.0, - "51": 764269056.0, - "52": 764269056.0, - "53": 764269056.0, - "54": 764269056.0, - "55": 764269056.0, - "56": 764269056.0, - "57": 764269056.0, - "58": 764269056.0, - "59": 764269056.0, - "60": 764269056.0, - "61": 764269056.0, - "62": 764269056.0, - "63": 764269056.0, - "64": 764269056.0, - "65": 764269056.0, + "1": 765317632.0, + "2": 765317632.0, + "3": 765317632.0, + "4": 765317632.0, + "5": 765317632.0, + "6": 765317632.0, + "7": 765317632.0, + "8": 765317632.0, + "9": 765317632.0, + "10": 765317632.0, + "11": 765317632.0, + "12": 765317632.0, + "13": 765317632.0, + "14": 765317632.0, + "15": 765317632.0, + "16": 765317632.0, + "17": 765317632.0, + "18": 765317632.0, + "19": 765317632.0, + "20": 765317632.0, + "21": 765317632.0, + "22": 765317632.0, + "23": 765317632.0, + "24": 765317632.0, + "25": 765317632.0, + "26": 765317632.0, + "27": 765317632.0, + "28": 765317632.0, + "29": 765317632.0, + "30": 765317632.0, + "31": 765317632.0, + "32": 765317632.0, + "33": 765317632.0, + "34": 765317632.0, + "35": 765317632.0, + "36": 765317632.0, + "37": 765317632.0, + "38": 765317632.0, + "39": 765317632.0, + "40": 765317632.0, + "41": 765317632.0, + "42": 765317632.0, + "43": 765317632.0, + "44": 765317632.0, + "45": 765317632.0, + "46": 765317632.0, + "47": 765317632.0, + "48": 765317632.0, + "49": 765317632.0, + "50": 765317632.0, + "51": 765317632.0, + "52": 765317632.0, + "53": 765317632.0, + "54": 765317632.0, + "55": 765317632.0, + "56": 765317632.0, + "57": 765317632.0, + "58": 765317632.0, + "59": 765317632.0, + "60": 765317632.0, + "61": 765317632.0, + "62": 765317632.0, + "63": 765317632.0, + "64": 765317632.0, + "65": 765317632.0, "66": 765317632.0, - "67": 764269056.0, - "68": 764269056.0, - "69": 764269056.0, - "70": 764269056.0, - "71": 764269056.0, - "72": 764269056.0, - "73": 764269056.0, + "67": 765317632.0, + "68": 765317632.0, + "69": 765317632.0, + "70": 765317632.0, + "71": 765317632.0, + "72": 765317632.0, + "73": 765317632.0, "74": 765317632.0, - "75": 764269056.0, - "76": 764269056.0, - "77": 764269056.0, - "78": 764269056.0, - "79": 764269056.0, - "80": 764269056.0, - "81": 764269056.0, - "82": 764269056.0, - "83": 764269056.0, - "84": 764269056.0, - "85": 764269056.0, - "86": 764269056.0, - "87": 764269056.0, - "88": 764269056.0, - "89": 764269056.0, - "90": 764269056.0, - "91": 764269056.0, - "92": 764269056.0, - "93": 764269056.0, - "94": 764269056.0, - "95": 764269056.0, - "96": 764269056.0, - "97": 764269056.0, - "98": 764269056.0, - "99": 764269056.0, - "100": 764269056.0 + "75": 765317632.0, + "76": 765317632.0, + "77": 765317632.0, + "78": 765317632.0, + "79": 765317632.0, + "80": 765317632.0, + "81": 765317632.0, + "82": 765317632.0, + "83": 765317632.0, + "84": 765317632.0, + "85": 765317632.0, + "86": 765317632.0, + "87": 765317632.0, + "88": 765317632.0, + "89": 765317632.0, + "90": 765317632.0, + "91": 765317632.0, + "92": 765317632.0, + "93": 765317632.0, + "94": 765317632.0, + "95": 765317632.0, + "96": 765317632.0, + "97": 765317632.0, + "98": 765317632.0, + "99": 765317632.0, + "100": 765317632.0 } }, "mem-max-allocated-bytes": { @@ -331,100 +331,100 @@ "4": 2683671552.0, "5": 2683671552.0, "6": 2683671552.0, - "7": 2683671552.0, - "8": 2683671552.0, - "9": 2683671552.0, - "10": 2683671552.0, - "11": 2683671552.0, - "12": 2683671552.0, - "13": 2683671552.0, - "14": 2683671552.0, - "15": 2683671552.0, - "16": 2683671552.0, - "17": 2683671552.0, - "18": 2683671552.0, - "19": 2683671552.0, - "20": 2683671552.0, - "21": 2683671552.0, - "22": 2683671552.0, - "23": 2683671552.0, - "24": 2684456448.0, - "25": 2684456448.0, - "26": 2684456448.0, - "27": 2684456448.0, - "28": 2684456448.0, - "29": 2684456448.0, - "30": 2684456448.0, - "31": 2684456448.0, - "32": 2684456448.0, - "33": 2684456448.0, - "34": 2684456448.0, - "35": 2684456448.0, - "36": 2684456448.0, - "37": 2684456448.0, - "38": 2684456448.0, - "39": 2684456448.0, - "40": 2684456448.0, - "41": 2684456448.0, - "42": 2684456448.0, - "43": 2684456448.0, - "44": 2684456448.0, - "45": 2684456448.0, - "46": 2684456448.0, - "47": 2684456448.0, - "48": 2684456448.0, - "49": 2684456448.0, - "50": 2684456448.0, - "51": 2684456448.0, - "52": 2684456448.0, - "53": 2684456448.0, - "54": 2684456448.0, - "55": 2684456448.0, - "56": 2684456448.0, - "57": 2684456448.0, - "58": 2684456448.0, - "59": 2684456448.0, - "60": 2684456448.0, - "61": 2684456448.0, - "62": 2684456448.0, - "63": 2684456448.0, - "64": 2684456448.0, - "65": 2684456448.0, - "66": 2684456448.0, - "67": 2684456448.0, - "68": 2684456448.0, - "69": 2684456448.0, - "70": 2684456448.0, - "71": 2684456448.0, - "72": 2684456448.0, - "73": 2684456448.0, - "74": 2684456448.0, - "75": 2684456448.0, - "76": 2684456448.0, - "77": 2684456448.0, - "78": 2684456448.0, - "79": 2684456448.0, - "80": 2684456448.0, - "81": 2684456448.0, - "82": 2684456448.0, - "83": 2684456448.0, - "84": 2684456448.0, - "85": 2684456448.0, - "86": 2684456448.0, - "87": 2684456448.0, - "88": 2684456448.0, - "89": 2684456448.0, - "90": 2684456448.0, - "91": 2684456448.0, - "92": 2684456448.0, - "93": 2684456448.0, - "94": 2684456448.0, - "95": 2684456448.0, - "96": 2684456448.0, - "97": 2684456448.0, - "98": 2684456448.0, - "99": 2684456448.0, - "100": 2684456448.0 + "7": 2684195840.0, + "8": 2684195840.0, + "9": 2684195840.0, + "10": 2684195840.0, + "11": 2684195840.0, + "12": 2684195840.0, + "13": 2684195840.0, + "14": 2684195840.0, + "15": 2684195840.0, + "16": 2684195840.0, + "17": 2684195840.0, + "18": 2684195840.0, + "19": 2684195840.0, + "20": 2684195840.0, + "21": 2684195840.0, + "22": 2684195840.0, + "23": 2684195840.0, + "24": 2684195840.0, + "25": 2684195840.0, + "26": 2684195840.0, + "27": 2684195840.0, + "28": 2684195840.0, + "29": 2684195840.0, + "30": 2684195840.0, + "31": 2684195840.0, + "32": 2684195840.0, + "33": 2684195840.0, + "34": 2684195840.0, + "35": 2684195840.0, + "36": 2684195840.0, + "37": 2684195840.0, + "38": 2684195840.0, + "39": 2684195840.0, + "40": 2684195840.0, + "41": 2684980736.0, + "42": 2684980736.0, + "43": 2684980736.0, + "44": 2684980736.0, + "45": 2684980736.0, + "46": 2684980736.0, + "47": 2684980736.0, + "48": 2684980736.0, + "49": 2684980736.0, + "50": 2684980736.0, + "51": 2684980736.0, + "52": 2684980736.0, + "53": 2684980736.0, + "54": 2684980736.0, + "55": 2684980736.0, + "56": 2684980736.0, + "57": 2684980736.0, + "58": 2684980736.0, + "59": 2684980736.0, + "60": 2684980736.0, + "61": 2684980736.0, + "62": 2684980736.0, + "63": 2684980736.0, + "64": 2684980736.0, + "65": 2684980736.0, + "66": 2684980736.0, + "67": 2684980736.0, + "68": 2684980736.0, + "69": 2684980736.0, + "70": 2684980736.0, + "71": 2684980736.0, + "72": 2684980736.0, + "73": 2684980736.0, + "74": 2684980736.0, + "75": 2684980736.0, + "76": 2684980736.0, + "77": 2684980736.0, + "78": 2684980736.0, + "79": 2684980736.0, + "80": 2684980736.0, + "81": 2684980736.0, + "82": 2684980736.0, + "83": 2684980736.0, + "84": 2684980736.0, + "85": 2684980736.0, + "86": 2684980736.0, + "87": 2684980736.0, + "88": 2684980736.0, + "89": 2684980736.0, + "90": 2684980736.0, + "91": 2684980736.0, + "92": 2684980736.0, + "93": 2684980736.0, + "94": 2684980736.0, + "95": 2684980736.0, + "96": 2684980736.0, + "97": 2684980736.0, + "98": 2684980736.0, + "99": 2684980736.0, + "100": 2684980736.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.22036, - "3": 2.20415, - "4": 2.08777, - "5": 1.57617, - "6": 1.98982, - "7": 1.83939, - "8": 2.41823, - "9": 2.12984, - "10": 2.15958, - "11": 1.93646, - "12": 1.88825, - "13": 2.04226, - "14": 2.39522, - "15": 1.80906, - "16": 2.04254, - "17": 2.1028, - "18": 1.76417, - "19": 2.03236, - "20": 1.66314, - "21": 2.27582, - "22": 1.80755, - "23": 2.17162, - "24": 1.65212, - "25": 2.34768, - "26": 2.56374, - "27": 1.92359, - "28": 1.75954, - "29": 1.50854, - "30": 2.00136, - "31": 1.56852, - "32": 2.69666, - "33": 1.51766, - "34": 1.51194, - "35": 2.10968, - "36": 2.32577, - "37": 1.74564, - "38": 1.57205, - "39": 2.01796, - "40": 2.09859, - "41": 2.06025, - "42": 2.78705, - "43": 1.90579, - "44": 1.77323, - "45": 2.54723, - "46": 1.82766, - "47": 1.99145, - "48": 2.38637, - "49": 1.82776, - "50": 2.13381, - "51": 1.2664, - "52": 2.02135, - "53": 2.09316, - "54": 1.91073, - "55": 1.95887, - "56": 2.7439, - "57": 1.51213, - "58": 2.2268, - "59": 1.59326, - "60": 1.78325, - "61": 2.143, - "62": 2.24637, - "63": 1.77571, - "64": 2.08826, - "65": 1.88434, - "66": 2.2212, - "67": 2.18085, - "68": 1.68225, - "69": 2.26237, - "70": 1.89611, - "71": 2.41413, - "72": 1.8583, - "73": 1.85479, - "74": 1.7717, - "75": 1.83759, - "76": 1.98578, - "77": 1.86957, - "78": 1.7902, - "79": 2.30534, - "80": 1.99186, - "81": 2.46469, - "82": 2.251, - "83": 2.0939, - "84": 1.81168, - "85": 1.93932, - "86": 1.81157, - "87": 2.37052, - "88": 2.10101, - "89": 2.39261, - "90": 2.02708, - "91": 2.08531, - "92": 2.34193, - "93": 1.74529, - "94": 2.20042, - "95": 1.95062, - "96": 1.87083, - "97": 1.94129, - "98": 2.13306, - "99": 2.61932, - "100": 1.86229 + "2": 6.91801, + "3": 2.01538, + "4": 2.12493, + "5": 1.74711, + "6": 1.74448, + "7": 1.90703, + "8": 2.29592, + "9": 2.08729, + "10": 2.02294, + "11": 2.11442, + "12": 1.70111, + "13": 2.07923, + "14": 2.346, + "15": 1.71593, + "16": 1.97503, + "17": 2.45652, + "18": 2.10059, + "19": 2.21036, + "20": 1.75941, + "21": 2.07408, + "22": 1.90841, + "23": 2.25775, + "24": 1.90654, + "25": 2.18589, + "26": 2.57321, + "27": 1.70065, + "28": 1.94398, + "29": 1.6833, + "30": 1.9207, + "31": 1.78808, + "32": 2.96668, + "33": 1.54881, + "34": 1.72445, + "35": 2.33087, + "36": 2.12756, + "37": 1.93292, + "38": 1.45961, + "39": 2.04643, + "40": 2.10393, + "41": 2.14195, + "42": 2.44302, + "43": 1.82695, + "44": 1.82888, + "45": 2.35373, + "46": 1.86203, + "47": 2.09528, + "48": 2.97147, + "49": 2.0905, + "50": 2.43283, + "51": 1.93189, + "52": 2.36368, + "53": 2.05511, + "54": 2.50922, + "55": 2.10164, + "56": 2.52909, + "57": 1.72307, + "58": 2.38429, + "59": 1.93733, + "60": 1.78587, + "61": 2.36871, + "62": 2.4657, + "63": 1.80161, + "64": 2.0395, + "65": 1.69979, + "66": 2.19227, + "67": 2.25439, + "68": 1.49545, + "69": 1.78357, + "70": 1.92554, + "71": 2.33859, + "72": 1.86341, + "73": 1.86907, + "74": 1.80828, + "75": 1.87276, + "76": 3.11067, + "77": 1.92833, + "78": 2.16611, + "79": 2.50582, + "80": 2.26898, + "81": 2.37451, + "82": 2.49786, + "83": 2.28095, + "84": 1.99303, + "85": 2.24139, + "86": 2.28918, + "87": 3.05868, + "88": 2.42441, + "89": 2.4392, + "90": 2.20925, + "91": 2.35881, + "92": 2.50724, + "93": 2.17204, + "94": 2.30156, + "95": 2.18908, + "96": 2.02642, + "97": 1.88456, + "98": 2.06743, + "99": 2.45409, + "100": 1.68709 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json index a1335d718ac..ba1251e1c87 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.96464, - "2": 10.95235, - "3": 10.95854, - "4": 10.95197, - "5": 10.95261, + "1": 10.96462, + "2": 10.95232, + "3": 10.95859, + "4": 10.95192, + "5": 10.95262, "6": 10.95382, - "7": 10.95159, - "8": 10.95132, - "9": 10.9587, - "10": 10.94456, - "11": 10.9368, - "12": 10.94339, + "7": 10.95158, + "8": 10.95135, + "9": 10.95869, + "10": 10.94452, + "11": 10.93685, + "12": 10.94335, "13": 10.94312, - "14": 10.94291, - "15": 10.91494, - "16": 10.90032, - "17": 10.91109, - "18": 10.90619, - "19": 10.90162, - "20": 10.80791, - "21": 10.7946, - "22": 10.80473, - "23": 10.78976, - "24": 10.77499, + "14": 10.94292, + "15": 10.9149, + "16": 10.90029, + "17": 10.91107, + "18": 10.90616, + "19": 10.90165, + "20": 10.80787, + "21": 10.79457, + "22": 10.80477, + "23": 10.78977, + "24": 10.77504, "25": 10.76169, - "26": 10.7552, - "27": 10.71953, - "28": 10.63914, - "29": 10.60852, - "30": 10.5865, - "31": 10.59099, - "32": 10.57106, - "33": 10.54032, - "34": 10.49898, - "35": 10.5003, - "36": 10.48632, - "37": 10.44935, - "38": 10.45072, - "39": 10.41769, - "40": 10.39539, - "41": 10.37479, - "42": 10.36062, - "43": 10.33044, - "44": 10.31902, - "45": 10.31601, - "46": 10.2765, - "47": 10.26822, - "48": 10.21836, - "49": 10.21385, - "50": 10.22195, - "51": 10.22215, - "52": 10.16751, + "26": 10.75515, + "27": 10.71954, + "28": 10.63906, + "29": 10.60855, + "30": 10.58656, + "31": 10.59096, + "32": 10.57113, + "33": 10.54034, + "34": 10.49902, + "35": 10.50031, + "36": 10.48629, + "37": 10.44938, + "38": 10.45069, + "39": 10.41772, + "40": 10.39543, + "41": 10.37489, + "42": 10.3607, + "43": 10.33047, + "44": 10.31907, + "45": 10.31615, + "46": 10.27658, + "47": 10.2683, + "48": 10.21832, + "49": 10.21392, + "50": 10.22203, + "51": 10.22219, + "52": 10.16759, "53": 10.17225, - "54": 10.13799, - "55": 10.10897, - "56": 10.13221, - "57": 10.12138, - "58": 10.12632, - "59": 10.07038, + "54": 10.138, + "55": 10.10895, + "56": 10.13222, + "57": 10.12137, + "58": 10.1263, + "59": 10.07041, "60": 10.09308, - "61": 10.04909, - "62": 10.01779, - "63": 10.08388, - "64": 10.03344, - "65": 10.00558, - "66": 10.04741, - "67": 10.02, - "68": 9.987, - "69": 9.99635, - "70": 9.98279, - "71": 10.00545, - "72": 9.99248, - "73": 9.97613, - "74": 9.96754, - "75": 9.93706, - "76": 9.96639, - "77": 9.96326, - "78": 9.91485, - "79": 9.91543, - "80": 9.92964, - "81": 9.95581, - "82": 9.89449, - "83": 9.85964, - "84": 9.80305, - "85": 9.7843, - "86": 9.88635, - "87": 9.90911, - "88": 9.88155, - "89": 9.82172, - "90": 9.82085, - "91": 9.82113, - "92": 9.81672, - "93": 9.74868, - "94": 9.82361, - "95": 9.81096, - "96": 9.79715, - "97": 9.74557, - "98": 9.76562, + "61": 10.0491, + "62": 10.01776, + "63": 10.08398, + "64": 10.03346, + "65": 10.0056, + "66": 10.04742, + "67": 10.02004, + "68": 9.98703, + "69": 9.99636, + "70": 9.98284, + "71": 10.0055, + "72": 9.99252, + "73": 9.9762, + "74": 9.9676, + "75": 9.93711, + "76": 9.96641, + "77": 9.96329, + "78": 9.91488, + "79": 9.9155, + "80": 9.92971, + "81": 9.95593, + "82": 9.89454, + "83": 9.85966, + "84": 9.80309, + "85": 9.78436, + "86": 9.88639, + "87": 9.90914, + "88": 9.88161, + "89": 9.82179, + "90": 9.82092, + "91": 9.82117, + "92": 9.81674, + "93": 9.7487, + "94": 9.82366, + "95": 9.81101, + "96": 9.7972, + "97": 9.74566, + "98": 9.76567, "99": 9.82205, - "100": 9.7044 + "100": 9.70445 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1876.0, - "2": 1608.0, - "3": 1787.0, - "4": 1675.0, - "5": 1820.0, - "6": 1678.0, - "7": 1861.0, - "8": 1749.0, - "9": 1745.0, - "10": 1830.0, - "11": 1702.0, - "12": 1714.0, - "13": 1843.0, - "14": 1903.0, - "15": 1605.0, - "16": 1760.0, - "17": 1770.0, - "18": 1856.0, - "19": 1689.0, - "20": 1673.0, - "21": 1780.0, - "22": 1786.0, - "23": 1719.0, - "24": 1842.0, + "1": 1759.0, + "2": 1644.0, + "3": 1736.0, + "4": 1783.0, + "5": 1782.0, + "6": 1759.0, + "7": 1975.0, + "8": 1703.0, + "9": 1724.0, + "10": 1760.0, + "11": 1678.0, + "12": 1698.0, + "13": 1776.0, + "14": 1809.0, + "15": 1596.0, + "16": 1706.0, + "17": 1765.0, + "18": 1843.0, + "19": 1667.0, + "20": 1696.0, + "21": 1746.0, + "22": 1714.0, + "23": 1724.0, + "24": 1912.0, "25": 1681.0, - "26": 1756.0, - "27": 1825.0, - "28": 1791.0, - "29": 1899.0, - "30": 1804.0, - "31": 2003.0, - "32": 1912.0, - "33": 1909.0, - "34": 1958.0, - "35": 2028.0, - "36": 1995.0, - "37": 2139.0, - "38": 2014.0, - "39": 2140.0, - "40": 2249.0, - "41": 2296.0, - "42": 2037.0, - "43": 2374.0, - "44": 2154.0, - "45": 2438.0, - "46": 2301.0, - "47": 2380.0, - "48": 2587.0, - "49": 2886.0, - "50": 2506.0, - "51": 2500.0, - "52": 2612.0, - "53": 2596.0, - "54": 2645.0, - "55": 2384.0, - "56": 2591.0, - "57": 2228.0, - "58": 3517.0, - "59": 2897.0, - "60": 2820.0, - "61": 2689.0, - "62": 3146.0, - "63": 3286.0, - "64": 3553.0, - "65": 2726.0, - "66": 3019.0, - "67": 3723.0, - "68": 3358.0, - "69": 2938.0, - "70": 3348.0, - "71": 3156.0, - "72": 2903.0, - "73": 3341.0, - "74": 3310.0, - "75": 3004.0, - "76": 3243.0, - "77": 3670.0, - "78": 3232.0, - "79": 3347.0, - "80": 3054.0, - "81": 3333.0, - "82": 2884.0, - "83": 3068.0, - "84": 2960.0, - "85": 2697.0, - "86": 3154.0, - "87": 2779.0, - "88": 3104.0, - "89": 3265.0, - "90": 3658.0, - "91": 2837.0, - "92": 3002.0, - "93": 3244.0, - "94": 2880.0, - "95": 3420.0, - "96": 3321.0, - "97": 3418.0, - "98": 3433.0, - "99": 3112.0, - "100": 2976.0 + "26": 1767.0, + "27": 1717.0, + "28": 1747.0, + "29": 1921.0, + "30": 1819.0, + "31": 1989.0, + "32": 1974.0, + "33": 1941.0, + "34": 1983.0, + "35": 2002.0, + "36": 1960.0, + "37": 2165.0, + "38": 2149.0, + "39": 2143.0, + "40": 2211.0, + "41": 2279.0, + "42": 2030.0, + "43": 2382.0, + "44": 2162.0, + "45": 2505.0, + "46": 2341.0, + "47": 2333.0, + "48": 2479.0, + "49": 2706.0, + "50": 2461.0, + "51": 2429.0, + "52": 2701.0, + "53": 2565.0, + "54": 2688.0, + "55": 2484.0, + "56": 2632.0, + "57": 2193.0, + "58": 3499.0, + "59": 2862.0, + "60": 3006.0, + "61": 2688.0, + "62": 3002.0, + "63": 3164.0, + "64": 3662.0, + "65": 2676.0, + "66": 3024.0, + "67": 3772.0, + "68": 3319.0, + "69": 3042.0, + "70": 3390.0, + "71": 3166.0, + "72": 2995.0, + "73": 3439.0, + "74": 3392.0, + "75": 3090.0, + "76": 3165.0, + "77": 3627.0, + "78": 3196.0, + "79": 3207.0, + "80": 3070.0, + "81": 3444.0, + "82": 2883.0, + "83": 3009.0, + "84": 2976.0, + "85": 2720.0, + "86": 3135.0, + "87": 2819.0, + "88": 3175.0, + "89": 3208.0, + "90": 3821.0, + "91": 2897.0, + "92": 2937.0, + "93": 3131.0, + "94": 2913.0, + "95": 3323.0, + "96": 3429.0, + "97": 3423.0, + "98": 3370.0, + "99": 3018.0, + "100": 3064.0 } }, "mem-allocated-bytes": { @@ -279,13 +279,13 @@ "59": 551148032.0, "60": 551148032.0, "61": 551148032.0, - "62": 551148032.0, + "62": 552065536.0, "63": 551148032.0, "64": 551148032.0, "65": 551148032.0, "66": 551148032.0, "67": 551148032.0, - "68": 551148032.0, + "68": 552065536.0, "69": 551148032.0, "70": 551148032.0, "71": 551148032.0, @@ -297,7 +297,7 @@ "77": 551148032.0, "78": 551148032.0, "79": 551148032.0, - "80": 551148032.0, + "80": 552065536.0, "81": 551148032.0, "82": 551148032.0, "83": 551148032.0, @@ -305,8 +305,8 @@ "85": 551148032.0, "86": 551148032.0, "87": 551148032.0, - "88": 552065536.0, - "89": 552065536.0, + "88": 551148032.0, + "89": 551148032.0, "90": 551148032.0, "91": 551148032.0, "92": 551148032.0, @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.13132, - "3": 0.11546, - "4": 0.11539, - "5": 0.11281, - "6": 0.11373, - "7": 0.11124, - "8": 0.11125, - "9": 0.11028, - "10": 0.11267, - "11": 0.11493, - "12": 0.11184, - "13": 0.11457, - "14": 0.11048, - "15": 0.10982, - "16": 0.1125, - "17": 0.11157, - "18": 0.11451, - "19": 0.11475, - "20": 0.11218, - "21": 0.11182, - "22": 0.11475, - "23": 0.11115, - "24": 0.11448, - "25": 0.11651, - "26": 0.11267, - "27": 0.11609, - "28": 0.11474, - "29": 0.11569, - "30": 0.11931, - "31": 0.11221, - "32": 0.11228, - "33": 0.11303, - "34": 0.11298, - "35": 0.11264, - "36": 0.11292, - "37": 0.11259, - "38": 0.11129, - "39": 0.11349, - "40": 0.11417, - "41": 0.11467, - "42": 0.11487, - "43": 0.11303, - "44": 0.1153, - "45": 0.11421, - "46": 0.11423, - "47": 0.11489, - "48": 0.11482, - "49": 0.11902, - "50": 0.11667, - "51": 0.24999, - "52": 0.14548, - "53": 0.11773, - "54": 0.11384, - "55": 0.1174, - "56": 0.11969, - "57": 0.1174, - "58": 0.1164, - "59": 0.11699, - "60": 0.11224, - "61": 0.11664, - "62": 0.1149, - "63": 0.11383, - "64": 0.11567, - "65": 0.11293, - "66": 0.11406, - "67": 0.11323, - "68": 0.11436, - "69": 0.11367, - "70": 0.11422, - "71": 0.11373, - "72": 0.12142, - "73": 0.12045, - "74": 0.12039, - "75": 0.12131, - "76": 0.11987, - "77": 0.11191, - "78": 0.11544, - "79": 0.11753, - "80": 0.11503, - "81": 0.11188, - "82": 0.11582, - "83": 0.11361, - "84": 0.11973, - "85": 0.12213, - "86": 0.11643, - "87": 0.12525, - "88": 0.12257, - "89": 0.11041, - "90": 0.11059, - "91": 0.10862, - "92": 0.11213, - "93": 0.11143, - "94": 0.11018, - "95": 0.11348, - "96": 0.11227, - "97": 0.11254, - "98": 0.11071, - "99": 0.10993, - "100": 0.11268 + "2": 7.92769, + "3": 0.11474, + "4": 0.1169, + "5": 0.11365, + "6": 0.11496, + "7": 0.11422, + "8": 0.11314, + "9": 0.11087, + "10": 0.11333, + "11": 0.11509, + "12": 0.11069, + "13": 0.11372, + "14": 0.11038, + "15": 0.11335, + "16": 0.11201, + "17": 0.1158, + "18": 0.11141, + "19": 0.11449, + "20": 0.1109, + "21": 0.11071, + "22": 0.11278, + "23": 0.11904, + "24": 0.10992, + "25": 0.11365, + "26": 0.11174, + "27": 0.11402, + "28": 0.11278, + "29": 0.11279, + "30": 0.11241, + "31": 0.11398, + "32": 0.11209, + "33": 0.12066, + "34": 0.11331, + "35": 0.11605, + "36": 0.11513, + "37": 0.11823, + "38": 0.1173, + "39": 0.11804, + "40": 0.11361, + "41": 0.1126, + "42": 0.11443, + "43": 0.1155, + "44": 0.11792, + "45": 0.11529, + "46": 0.11406, + "47": 0.11243, + "48": 0.11534, + "49": 0.11767, + "50": 0.11371, + "51": 0.1501, + "52": 0.14592, + "53": 0.12139, + "54": 0.12061, + "55": 0.11138, + "56": 0.11316, + "57": 0.11256, + "58": 0.11436, + "59": 0.11347, + "60": 0.10889, + "61": 0.11151, + "62": 0.11246, + "63": 0.11916, + "64": 0.11125, + "65": 0.11055, + "66": 0.11074, + "67": 0.10978, + "68": 0.11184, + "69": 0.10956, + "70": 0.10924, + "71": 0.11186, + "72": 0.11659, + "73": 0.11747, + "74": 0.10997, + "75": 0.10976, + "76": 0.11583, + "77": 0.11313, + "78": 0.11607, + "79": 0.11279, + "80": 0.11306, + "81": 0.11038, + "82": 0.11268, + "83": 0.1116, + "84": 0.11016, + "85": 0.11199, + "86": 0.10932, + "87": 0.11333, + "88": 0.11236, + "89": 0.11091, + "90": 0.11295, + "91": 0.11135, + "92": 0.11013, + "93": 0.11919, + "94": 0.10798, + "95": 0.1131, + "96": 0.11397, + "97": 0.11309, + "98": 0.11004, + "99": 0.10986, + "100": 0.11328 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json index c860b494165..5b1712f2f10 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93832, - "2": 10.93769, - "3": 10.93625, - "4": 10.9423, - "5": 10.93352, - "6": 10.93273, - "7": 10.93793, - "8": 10.93357, - "9": 10.93553, - "10": 10.93697, - "11": 10.92587, - "12": 10.92415, + "1": 10.93827, + "2": 10.93772, + "3": 10.93628, + "4": 10.94226, + "5": 10.93351, + "6": 10.93269, + "7": 10.93786, + "8": 10.93354, + "9": 10.93555, + "10": 10.93689, + "11": 10.92583, + "12": 10.92411, "13": 10.92509, - "14": 10.9126, + "14": 10.91251, "15": 10.89336, - "16": 10.88824, - "17": 10.89973, - "18": 10.87823, - "19": 10.87587, - "20": 10.79123, - "21": 10.78019, - "22": 10.77476, - "23": 10.77499, + "16": 10.88832, + "17": 10.89971, + "18": 10.87815, + "19": 10.87589, + "20": 10.79125, + "21": 10.78018, + "22": 10.77478, + "23": 10.77507, "24": 10.73642, - "25": 10.74387, + "25": 10.74386, "26": 10.72466, "27": 10.6963, - "28": 10.62351, - "29": 10.6038, - "30": 10.58045, - "31": 10.56963, - "32": 10.54858, - "33": 10.52048, - "34": 10.48554, - "35": 10.49228, - "36": 10.4709, - "37": 10.43637, - "38": 10.44107, - "39": 10.40279, - "40": 10.39284, - "41": 10.36864, - "42": 10.34611, - "43": 10.32631, - "44": 10.28964, - "45": 10.30829, - "46": 10.26822, - "47": 10.25249, - "48": 10.2037, - "49": 10.20119, - "50": 10.20925, - "51": 10.2053, - "52": 10.15734, - "53": 10.16687, - "54": 10.13043, - "55": 10.10057, - "56": 10.13011, - "57": 10.11464, - "58": 10.13007, + "28": 10.62356, + "29": 10.60383, + "30": 10.58046, + "31": 10.56962, + "32": 10.54867, + "33": 10.52059, + "34": 10.48557, + "35": 10.4923, + "36": 10.47098, + "37": 10.43638, + "38": 10.44115, + "39": 10.40285, + "40": 10.39287, + "41": 10.36868, + "42": 10.34615, + "43": 10.3263, + "44": 10.28968, + "45": 10.30834, + "46": 10.26832, + "47": 10.25254, + "48": 10.20377, + "49": 10.20127, + "50": 10.20928, + "51": 10.20532, + "52": 10.15741, + "53": 10.16688, + "54": 10.13045, + "55": 10.10059, + "56": 10.13012, + "57": 10.11465, + "58": 10.13009, "59": 10.07596, - "60": 10.08697, - "61": 10.04223, - "62": 10.00953, - "63": 10.08368, - "64": 10.03719, - "65": 10.00997, - "66": 10.04022, - "67": 10.00689, - "68": 9.97342, - "69": 9.99439, - "70": 9.98223, - "71": 10.00185, - "72": 9.97365, - "73": 9.96699, - "74": 9.95725, - "75": 9.92477, + "60": 10.08704, + "61": 10.04224, + "62": 10.00959, + "63": 10.08375, + "64": 10.03718, + "65": 10.00996, + "66": 10.04027, + "67": 10.00691, + "68": 9.97351, + "69": 9.99451, + "70": 9.98228, + "71": 10.00188, + "72": 9.97372, + "73": 9.96702, + "74": 9.95733, + "75": 9.92481, "76": 9.96624, "77": 9.95575, - "78": 9.90173, - "79": 9.90898, - "80": 9.92106, - "81": 9.94319, - "82": 9.88581, - "83": 9.84494, - "84": 9.78312, - "85": 9.77444, - "86": 9.87683, - "87": 9.91005, - "88": 9.882, - "89": 9.81987, - "90": 9.80797, - "91": 9.81902, - "92": 9.80781, - "93": 9.74749, - "94": 9.82029, - "95": 9.8157, - "96": 9.80355, - "97": 9.7406, - "98": 9.76895, - "99": 9.8159, - "100": 9.70554 + "78": 9.90176, + "79": 9.90904, + "80": 9.9211, + "81": 9.94325, + "82": 9.88582, + "83": 9.84498, + "84": 9.78316, + "85": 9.77448, + "86": 9.87692, + "87": 9.91006, + "88": 9.88206, + "89": 9.81986, + "90": 9.80799, + "91": 9.81903, + "92": 9.80785, + "93": 9.74755, + "94": 9.82036, + "95": 9.81577, + "96": 9.80358, + "97": 9.74069, + "98": 9.76901, + "99": 9.81594, + "100": 9.70562 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1792.0, - "2": 1809.0, - "3": 1721.0, - "4": 1751.0, - "5": 1756.0, - "6": 1766.0, - "7": 2096.0, - "8": 1713.0, - "9": 1804.0, - "10": 1718.0, - "11": 1736.0, - "12": 1724.0, - "13": 1775.0, - "14": 1926.0, - "15": 1658.0, - "16": 1757.0, - "17": 1812.0, - "18": 1879.0, - "19": 1710.0, - "20": 1679.0, - "21": 1723.0, - "22": 1690.0, - "23": 1753.0, - "24": 1892.0, - "25": 1767.0, - "26": 1869.0, - "27": 1862.0, - "28": 1902.0, - "29": 1826.0, - "30": 1880.0, - "31": 1998.0, - "32": 1982.0, - "33": 1996.0, - "34": 2175.0, - "35": 2051.0, + "1": 1791.0, + "2": 1794.0, + "3": 1774.0, + "4": 1707.0, + "5": 1776.0, + "6": 1774.0, + "7": 2058.0, + "8": 1681.0, + "9": 1838.0, + "10": 1789.0, + "11": 1706.0, + "12": 1660.0, + "13": 1813.0, + "14": 1785.0, + "15": 1647.0, + "16": 1709.0, + "17": 1784.0, + "18": 1800.0, + "19": 1627.0, + "20": 1761.0, + "21": 1790.0, + "22": 1753.0, + "23": 1740.0, + "24": 1876.0, + "25": 1709.0, + "26": 1892.0, + "27": 1821.0, + "28": 1769.0, + "29": 1910.0, + "30": 1721.0, + "31": 2025.0, + "32": 1968.0, + "33": 2014.0, + "34": 1941.0, + "35": 2114.0, "36": 1974.0, - "37": 2256.0, - "38": 2159.0, - "39": 2063.0, - "40": 2239.0, - "41": 2379.0, - "42": 1944.0, - "43": 2319.0, - "44": 2172.0, - "45": 2550.0, - "46": 2381.0, - "47": 2453.0, - "48": 2591.0, - "49": 2636.0, - "50": 2590.0, - "51": 2477.0, - "52": 2793.0, - "53": 2561.0, - "54": 2717.0, - "55": 2472.0, - "56": 2677.0, - "57": 2183.0, - "58": 3495.0, - "59": 2935.0, - "60": 2895.0, - "61": 2757.0, - "62": 3041.0, - "63": 3137.0, - "64": 3332.0, - "65": 2609.0, - "66": 3019.0, - "67": 3698.0, - "68": 3370.0, - "69": 2933.0, - "70": 3278.0, - "71": 3092.0, - "72": 3047.0, - "73": 3310.0, - "74": 3031.0, - "75": 3167.0, - "76": 3175.0, - "77": 3706.0, - "78": 3237.0, - "79": 3195.0, - "80": 3060.0, - "81": 3281.0, - "82": 3156.0, - "83": 3213.0, - "84": 2976.0, - "85": 2832.0, - "86": 3080.0, - "87": 2995.0, - "88": 3282.0, - "89": 2781.0, - "90": 3476.0, - "91": 2998.0, - "92": 2978.0, - "93": 3222.0, - "94": 3236.0, - "95": 3461.0, - "96": 3503.0, - "97": 3660.0, - "98": 3618.0, - "99": 3139.0, - "100": 3325.0 + "37": 2192.0, + "38": 2098.0, + "39": 2200.0, + "40": 2326.0, + "41": 2322.0, + "42": 1973.0, + "43": 2366.0, + "44": 2124.0, + "45": 2480.0, + "46": 2376.0, + "47": 2474.0, + "48": 2571.0, + "49": 2729.0, + "50": 2520.0, + "51": 2535.0, + "52": 2643.0, + "53": 2500.0, + "54": 2804.0, + "55": 2465.0, + "56": 2642.0, + "57": 2279.0, + "58": 3437.0, + "59": 2900.0, + "60": 2882.0, + "61": 2663.0, + "62": 3087.0, + "63": 3166.0, + "64": 3453.0, + "65": 2565.0, + "66": 3047.0, + "67": 3793.0, + "68": 3427.0, + "69": 2948.0, + "70": 3106.0, + "71": 3002.0, + "72": 2919.0, + "73": 3334.0, + "74": 3171.0, + "75": 3022.0, + "76": 3089.0, + "77": 3742.0, + "78": 3125.0, + "79": 3174.0, + "80": 2973.0, + "81": 3351.0, + "82": 3108.0, + "83": 3161.0, + "84": 2969.0, + "85": 2740.0, + "86": 3109.0, + "87": 2940.0, + "88": 3162.0, + "89": 2814.0, + "90": 3576.0, + "91": 3124.0, + "92": 3001.0, + "93": 3157.0, + "94": 3170.0, + "95": 3476.0, + "96": 3481.0, + "97": 3689.0, + "98": 3612.0, + "99": 3224.0, + "100": 3437.0 } }, "mem-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 2398797312.0, - "2": 2681558016.0, - "3": 2681560064.0, - "4": 2681560064.0, - "5": 2682084352.0, - "6": 2682084352.0, - "7": 2682084352.0, - "8": 2682084352.0, - "9": 2682084352.0, - "10": 2682084352.0, - "11": 2682084352.0, - "12": 2682084352.0, - "13": 2682084352.0, - "14": 2682084352.0, - "15": 2682084352.0, - "16": 2682084352.0, - "17": 2682084352.0, - "18": 2682084352.0, - "19": 2682084352.0, - "20": 2682084352.0, - "21": 2682084352.0, - "22": 2682084352.0, - "23": 2682084352.0, - "24": 2682084352.0, - "25": 2682084352.0, - "26": 2682084352.0, - "27": 2682084352.0, - "28": 2682084352.0, - "29": 2682084352.0, - "30": 2682084352.0, - "31": 2682084352.0, - "32": 2682084352.0, - "33": 2682084352.0, - "34": 2682084352.0, - "35": 2682084352.0, - "36": 2682084352.0, - "37": 2682084352.0, - "38": 2682084352.0, - "39": 2682084352.0, - "40": 2682084352.0, - "41": 2682084352.0, - "42": 2682084352.0, - "43": 2682084352.0, - "44": 2682084352.0, - "45": 2682084352.0, - "46": 2682084352.0, - "47": 2682084352.0, - "48": 2682084352.0, - "49": 2682084352.0, - "50": 2682084352.0, - "51": 2682084352.0, - "52": 2682084352.0, - "53": 2682084352.0, - "54": 2682084352.0, - "55": 2682084352.0, - "56": 2682084352.0, - "57": 2682084352.0, - "58": 2682084352.0, - "59": 2682084352.0, - "60": 2682084352.0, - "61": 2682084352.0, - "62": 2682084352.0, - "63": 2682084352.0, - "64": 2682084352.0, - "65": 2682084352.0, - "66": 2682084352.0, - "67": 2682084352.0, - "68": 2682084352.0, - "69": 2682084352.0, - "70": 2682084352.0, - "71": 2682084352.0, - "72": 2682084352.0, - "73": 2682084352.0, - "74": 2682084352.0, - "75": 2682084352.0, - "76": 2682084352.0, - "77": 2682084352.0, - "78": 2682084352.0, - "79": 2682084352.0, - "80": 2682607104.0, - "81": 2682607104.0, - "82": 2682607104.0, - "83": 2682607104.0, - "84": 2682607104.0, - "85": 2682607104.0, - "86": 2682607104.0, - "87": 2682607104.0, - "88": 2682607104.0, - "89": 2682607104.0, - "90": 2682607104.0, - "91": 2682607104.0, - "92": 2682607104.0, - "93": 2682607104.0, - "94": 2682607104.0, - "95": 2682607104.0, - "96": 2682607104.0, - "97": 2682607104.0, - "98": 2682607104.0, - "99": 2682607104.0, - "100": 2682607104.0 + "2": 2679462912.0, + "3": 2679462912.0, + "4": 2679462912.0, + "5": 2679462912.0, + "6": 2679462912.0, + "7": 2679462912.0, + "8": 2679462912.0, + "9": 2679987200.0, + "10": 2679987200.0, + "11": 2679987200.0, + "12": 2679987200.0, + "13": 2679987200.0, + "14": 2679987200.0, + "15": 2679987200.0, + "16": 2679987200.0, + "17": 2679987200.0, + "18": 2679987200.0, + "19": 2679987200.0, + "20": 2679987200.0, + "21": 2679987200.0, + "22": 2679987200.0, + "23": 2679987200.0, + "24": 2679987200.0, + "25": 2679987200.0, + "26": 2679987200.0, + "27": 2679987200.0, + "28": 2679987200.0, + "29": 2679987200.0, + "30": 2679987200.0, + "31": 2679987200.0, + "32": 2679987200.0, + "33": 2679987200.0, + "34": 2679987200.0, + "35": 2679987200.0, + "36": 2679987200.0, + "37": 2679987200.0, + "38": 2679987200.0, + "39": 2679987200.0, + "40": 2679987200.0, + "41": 2679987200.0, + "42": 2679987200.0, + "43": 2679987200.0, + "44": 2679987200.0, + "45": 2679987200.0, + "46": 2679987200.0, + "47": 2679987200.0, + "48": 2679987200.0, + "49": 2679987200.0, + "50": 2679987200.0, + "51": 2679987200.0, + "52": 2679987200.0, + "53": 2679987200.0, + "54": 2679987200.0, + "55": 2679987200.0, + "56": 2679987200.0, + "57": 2679987200.0, + "58": 2679987200.0, + "59": 2679987200.0, + "60": 2679987200.0, + "61": 2679987200.0, + "62": 2679987200.0, + "63": 2679987200.0, + "64": 2679987200.0, + "65": 2679987200.0, + "66": 2679987200.0, + "67": 2679987200.0, + "68": 2680511488.0, + "69": 2680511488.0, + "70": 2680511488.0, + "71": 2680511488.0, + "72": 2680511488.0, + "73": 2680511488.0, + "74": 2680511488.0, + "75": 2680511488.0, + "76": 2680511488.0, + "77": 2680511488.0, + "78": 2680511488.0, + "79": 2680511488.0, + "80": 2680511488.0, + "81": 2680511488.0, + "82": 2680511488.0, + "83": 2680511488.0, + "84": 2680511488.0, + "85": 2680511488.0, + "86": 2680511488.0, + "87": 2680511488.0, + "88": 2680511488.0, + "89": 2680511488.0, + "90": 2680511488.0, + "91": 2680511488.0, + "92": 2680511488.0, + "93": 2680511488.0, + "94": 2680511488.0, + "95": 2680511488.0, + "96": 2680511488.0, + "97": 2680511488.0, + "98": 2680511488.0, + "99": 2680511488.0, + "100": 2680511488.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.46829, - "3": 2.19116, - "4": 1.95809, - "5": 1.40068, - "6": 1.74239, - "7": 1.78913, - "8": 2.40733, - "9": 2.03666, - "10": 1.41436, - "11": 1.81047, - "12": 1.54887, - "13": 1.6746, - "14": 1.41729, - "15": 1.49013, - "16": 1.65749, - "17": 2.31569, - "18": 1.48178, - "19": 1.74017, - "20": 1.59272, - "21": 1.66007, - "22": 1.18588, - "23": 1.65601, - "24": 1.46353, - "25": 2.08613, - "26": 2.0327, - "27": 1.58821, - "28": 1.55791, - "29": 1.31213, - "30": 1.58975, - "31": 1.5941, - "32": 2.52529, - "33": 1.30473, - "34": 1.55279, - "35": 1.88412, - "36": 1.68746, - "37": 1.61395, - "38": 1.53528, - "39": 1.87614, - "40": 1.51385, - "41": 1.63358, - "42": 2.17455, - "43": 1.85386, - "44": 1.75152, - "45": 1.99743, - "46": 1.99147, - "47": 1.67234, - "48": 1.79448, - "49": 1.33978, - "50": 1.72926, - "51": 1.60723, - "52": 1.86053, - "53": 1.67615, - "54": 1.65761, - "55": 1.60302, - "56": 2.41491, - "57": 1.29123, - "58": 1.91886, - "59": 1.50095, - "60": 1.44339, - "61": 1.91511, - "62": 2.41539, - "63": 1.53165, - "64": 2.01962, - "65": 1.96493, - "66": 1.97869, - "67": 1.71408, - "68": 1.48955, - "69": 1.54863, - "70": 1.59482, - "71": 2.13789, - "72": 1.64486, - "73": 1.55983, - "74": 1.65312, - "75": 1.58677, - "76": 1.7833, - "77": 1.67262, - "78": 1.81241, - "79": 2.13938, - "80": 1.6973, - "81": 2.00145, - "82": 1.95043, - "83": 1.66526, - "84": 1.38491, - "85": 1.75651, - "86": 1.57279, - "87": 1.96625, - "88": 2.06727, - "89": 1.736, - "90": 1.65592, - "91": 1.68149, - "92": 2.15955, - "93": 1.51492, - "94": 1.73084, - "95": 2.00253, - "96": 2.01546, - "97": 1.7338, - "98": 1.93187, - "99": 2.28739, - "100": 1.55635 + "2": 6.85715, + "3": 2.12919, + "4": 1.5445, + "5": 1.38735, + "6": 1.46162, + "7": 1.49336, + "8": 1.74709, + "9": 1.69804, + "10": 1.72936, + "11": 1.50098, + "12": 1.45975, + "13": 1.55511, + "14": 1.79173, + "15": 1.36068, + "16": 1.56857, + "17": 2.02909, + "18": 1.36914, + "19": 1.53725, + "20": 1.64153, + "21": 1.74595, + "22": 1.2907, + "23": 1.37499, + "24": 1.31017, + "25": 1.87529, + "26": 2.1228, + "27": 1.66725, + "28": 1.81367, + "29": 1.57279, + "30": 1.61824, + "31": 1.36993, + "32": 2.38613, + "33": 1.4071, + "34": 1.18678, + "35": 1.96598, + "36": 2.05754, + "37": 1.24701, + "38": 1.44914, + "39": 1.77165, + "40": 1.3593, + "41": 1.42317, + "42": 1.74293, + "43": 1.73397, + "44": 1.60176, + "45": 1.80056, + "46": 1.69855, + "47": 1.78211, + "48": 2.00202, + "49": 1.29374, + "50": 1.85501, + "51": 0.99846, + "52": 1.85972, + "53": 1.51875, + "54": 1.61615, + "55": 1.62922, + "56": 2.53025, + "57": 1.35595, + "58": 2.01776, + "59": 1.62558, + "60": 1.31808, + "61": 1.96109, + "62": 1.96072, + "63": 1.36181, + "64": 1.84859, + "65": 1.72181, + "66": 1.95957, + "67": 1.64965, + "68": 1.25299, + "69": 1.39168, + "70": 1.56355, + "71": 2.00957, + "72": 1.25252, + "73": 1.6479, + "74": 1.9527, + "75": 1.44947, + "76": 1.80405, + "77": 1.5815, + "78": 1.59153, + "79": 1.91663, + "80": 1.61818, + "81": 1.64839, + "82": 2.05066, + "83": 1.6762, + "84": 1.42813, + "85": 1.36891, + "86": 1.37032, + "87": 1.71854, + "88": 1.82867, + "89": 1.71827, + "90": 1.54231, + "91": 1.65523, + "92": 1.91089, + "93": 1.38134, + "94": 1.43821, + "95": 1.92994, + "96": 1.56414, + "97": 1.44047, + "98": 1.58043, + "99": 1.97684, + "100": 1.3584 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json index 9349fc45786..23708a2a714 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.94135, + "1": 10.94145, "2": 10.93474, - "3": 10.94379, - "4": 10.93302, - "5": 10.93416, - "6": 10.93731, - "7": 10.93993, - "8": 10.93567, - "9": 10.93345, - "10": 10.93259, + "3": 10.94378, + "4": 10.93299, + "5": 10.93415, + "6": 10.9373, + "7": 10.93992, + "8": 10.93571, + "9": 10.93336, + "10": 10.93261, "11": 10.92506, - "12": 10.91971, + "12": 10.91979, "13": 10.92588, - "14": 10.92627, - "15": 10.8888, - "16": 10.87344, - "17": 10.87977, - "18": 10.88077, - "19": 10.86265, - "20": 10.80654, - "21": 10.78325, - "22": 10.784, - "23": 10.77457, - "24": 10.75095, - "25": 10.7431, - "26": 10.72415, - "27": 10.67536, - "28": 10.60819, - "29": 10.59235, - "30": 10.57838, - "31": 10.573, - "32": 10.54372, - "33": 10.52272, - "34": 10.50051, - "35": 10.48199, - "36": 10.471, - "37": 10.44151, + "14": 10.9262, + "15": 10.88882, + "16": 10.87345, + "17": 10.87979, + "18": 10.88079, + "19": 10.86266, + "20": 10.80657, + "21": 10.78329, + "22": 10.78397, + "23": 10.77461, + "24": 10.75087, + "25": 10.74315, + "26": 10.72411, + "27": 10.67539, + "28": 10.60823, + "29": 10.59236, + "30": 10.57847, + "31": 10.57301, + "32": 10.54375, + "33": 10.52277, + "34": 10.50056, + "35": 10.48201, + "36": 10.47109, + "37": 10.44148, "38": 10.4108, - "39": 10.39247, - "40": 10.38024, - "41": 10.37505, - "42": 10.3336, - "43": 10.32979, + "39": 10.39246, + "40": 10.38027, + "41": 10.37514, + "42": 10.33363, + "43": 10.32989, "44": 10.27977, - "45": 10.2955, - "46": 10.27386, + "45": 10.29556, + "46": 10.27396, "47": 10.26477, - "48": 10.23709, - "49": 10.19637, - "50": 10.17891 + "48": 10.23704, + "49": 10.19644, + "50": 10.17897 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1304.0, - "2": 1321.0, - "3": 1205.0, - "4": 1267.0, - "5": 1308.0, - "6": 1342.0, - "7": 1261.0, - "8": 1430.0, - "9": 1254.0, - "10": 1155.0, - "11": 1324.0, - "12": 1186.0, - "13": 1197.0, - "14": 1338.0, - "15": 1193.0, - "16": 1187.0, - "17": 1187.0, - "18": 1226.0, - "19": 1225.0, - "20": 1212.0, - "21": 1289.0, - "22": 1327.0, - "23": 1249.0, - "24": 1198.0, - "25": 1367.0, - "26": 1317.0, - "27": 1307.0, - "28": 1397.0, - "29": 1349.0, - "30": 1446.0, - "31": 1446.0, - "32": 1403.0, - "33": 1387.0, - "34": 1371.0, - "35": 1552.0, - "36": 1482.0, - "37": 1483.0, - "38": 1529.0, - "39": 1569.0, - "40": 1592.0, - "41": 1447.0, - "42": 1530.0, - "43": 1664.0, - "44": 1588.0, - "45": 1607.0, - "46": 1866.0, - "47": 1756.0, - "48": 1881.0, - "49": 1787.0, - "50": 1847.0 + "1": 1256.0, + "2": 1260.0, + "3": 1227.0, + "4": 1248.0, + "5": 1225.0, + "6": 1348.0, + "7": 1246.0, + "8": 1272.0, + "9": 1245.0, + "10": 1241.0, + "11": 1328.0, + "12": 1243.0, + "13": 1211.0, + "14": 1330.0, + "15": 1234.0, + "16": 1132.0, + "17": 1241.0, + "18": 1251.0, + "19": 1188.0, + "20": 1189.0, + "21": 1282.0, + "22": 1305.0, + "23": 1306.0, + "24": 1262.0, + "25": 1346.0, + "26": 1250.0, + "27": 1269.0, + "28": 1374.0, + "29": 1347.0, + "30": 1491.0, + "31": 1418.0, + "32": 1392.0, + "33": 1405.0, + "34": 1328.0, + "35": 1459.0, + "36": 1515.0, + "37": 1541.0, + "38": 1513.0, + "39": 1637.0, + "40": 1569.0, + "41": 1522.0, + "42": 1628.0, + "43": 1700.0, + "44": 1590.0, + "45": 1629.0, + "46": 1824.0, + "47": 1771.0, + "48": 1863.0, + "49": 1725.0, + "50": 1864.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.97532, - "3": 2.47004, - "4": 1.72283, - "5": 2.23995, - "6": 2.20475, - "7": 2.30786, - "8": 2.66458, - "9": 1.91374, - "10": 2.01855, - "11": 2.45783, - "12": 1.7446, - "13": 2.14912, - "14": 2.64415, - "15": 1.38891, - "16": 2.13496, - "17": 2.09733, - "18": 1.78824, - "19": 1.99043, - "20": 2.11111, - "21": 2.2353, - "22": 1.86339, - "23": 2.0786, - "24": 1.60516, - "25": 1.88296, - "26": 2.61021, - "27": 1.49203, - "28": 2.18545, - "29": 2.23402, - "30": 1.77951, - "31": 1.89761, - "32": 2.27256, - "33": 2.06849, - "34": 2.99019, - "35": 2.22115, - "36": 2.30154, - "37": 2.13759, - "38": 2.36976, - "39": 2.08565, - "40": 2.42789, - "41": 1.9744, - "42": 1.98984, - "43": 2.0876, - "44": 1.94941, - "45": 2.94672, - "46": 1.81504, - "47": 2.30482, - "48": 1.73951, - "49": 2.55665, - "50": 2.48619 + "2": 6.3782, + "3": 2.64051, + "4": 1.67038, + "5": 2.0186, + "6": 1.91822, + "7": 2.18935, + "8": 2.5173, + "9": 1.77425, + "10": 1.66252, + "11": 1.99838, + "12": 1.73712, + "13": 1.80248, + "14": 2.33871, + "15": 1.63016, + "16": 2.16527, + "17": 1.80583, + "18": 1.83895, + "19": 1.8649, + "20": 2.00186, + "21": 2.5757, + "22": 2.06253, + "23": 2.46524, + "24": 1.87661, + "25": 1.99335, + "26": 2.92522, + "27": 2.09193, + "28": 2.28888, + "29": 2.33052, + "30": 1.99515, + "31": 1.7694, + "32": 2.15784, + "33": 1.96364, + "34": 2.72717, + "35": 2.15668, + "36": 1.95064, + "37": 1.64273, + "38": 1.95573, + "39": 1.71527, + "40": 2.35226, + "41": 1.88939, + "42": 1.7245, + "43": 2.13301, + "44": 1.99437, + "45": 2.50851, + "46": 1.94083, + "47": 2.23288, + "48": 1.58222, + "49": 1.98595, + "50": 2.42674 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json index fe6479caf7e..3c8f74fa731 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89814, - "2": 10.89631, - "3": 10.90187, - "4": 10.89348, - "5": 10.89713, - "6": 10.88447, - "7": 10.90724, - "8": 10.8899, - "9": 10.90482, - "10": 10.89557, - "11": 10.88482, - "12": 10.87805, - "13": 10.89171, - "14": 10.87219, + "1": 10.8982, + "2": 10.89629, + "3": 10.90191, + "4": 10.89353, + "5": 10.89714, + "6": 10.88452, + "7": 10.90719, + "8": 10.88997, + "9": 10.90487, + "10": 10.89562, + "11": 10.88494, + "12": 10.87811, + "13": 10.89176, + "14": 10.87214, "15": 10.85219, - "16": 10.84007, - "17": 10.85267, - "18": 10.83214, - "19": 10.83213, - "20": 10.74658, - "21": 10.74631, - "22": 10.7218, - "23": 10.70891, - "24": 10.68202, - "25": 10.67115, - "26": 10.66267, - "27": 10.60776, - "28": 10.55958, - "29": 10.52531, - "30": 10.49467, - "31": 10.4859, - "32": 10.472, - "33": 10.42622, - "34": 10.41733, - "35": 10.40437, - "36": 10.37776, - "37": 10.34605, - "38": 10.35576, - "39": 10.31893, - "40": 10.30851, + "16": 10.84008, + "17": 10.85268, + "18": 10.83216, + "19": 10.83211, + "20": 10.74654, + "21": 10.74629, + "22": 10.72187, + "23": 10.709, + "24": 10.68205, + "25": 10.67122, + "26": 10.66272, + "27": 10.60786, + "28": 10.55965, + "29": 10.52532, + "30": 10.49471, + "31": 10.48597, + "32": 10.47209, + "33": 10.42627, + "34": 10.41738, + "35": 10.40438, + "36": 10.37779, + "37": 10.34612, + "38": 10.35579, + "39": 10.31902, + "40": 10.30856, "41": 10.28993, - "42": 10.24612, - "43": 10.23762, - "44": 10.21154, - "45": 10.22062, - "46": 10.17855, + "42": 10.24616, + "43": 10.23767, + "44": 10.21156, + "45": 10.22066, + "46": 10.1786, "47": 10.17787, - "48": 10.12575, - "49": 10.12681, - "50": 10.1219 + "48": 10.1258, + "49": 10.1268, + "50": 10.12202 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22962392.0, - "2": 22849530.0, - "3": 22709824.0, - "4": 22777500.0, - "5": 22782868.0, - "6": 22745208.0, - "7": 22871616.0, - "8": 22611016.0, - "9": 22758184.0, - "10": 22488768.0, - "11": 22753326.0, - "12": 22640850.0, - "13": 23323068.0, - "14": 22992126.0, - "15": 22717790.0, - "16": 22822396.0, - "17": 22928836.0, - "18": 22994884.0, - "19": 23085022.0, - "20": 22723130.0, - "21": 22917994.0, - "22": 22944704.0, - "23": 22628748.0, - "24": 22862012.0, - "25": 22636090.0, - "26": 23004312.0, - "27": 22801808.0, - "28": 22998904.0, - "29": 22978984.0, - "30": 22943914.0, - "31": 22908222.0, - "32": 22665164.0, - "33": 22740194.0, - "34": 23076468.0, - "35": 22749952.0, - "36": 22694654.0, - "37": 23103940.0, - "38": 22964598.0, - "39": 22985086.0, - "40": 22750600.0, - "41": 23064904.0, - "42": 22688808.0, - "43": 22987284.0, - "44": 22704616.0, - "45": 22847632.0, - "46": 22733968.0, - "47": 22850384.0, - "48": 22833474.0, - "49": 22888972.0, - "50": 22644660.0 + "1": 22962456.0, + "2": 22849516.0, + "3": 22709852.0, + "4": 22777460.0, + "5": 22782948.0, + "6": 22745084.0, + "7": 22871700.0, + "8": 22610966.0, + "9": 22758244.0, + "10": 22488846.0, + "11": 22753404.0, + "12": 22640832.0, + "13": 23323176.0, + "14": 22991992.0, + "15": 22717880.0, + "16": 22822390.0, + "17": 22928926.0, + "18": 22994824.0, + "19": 23085090.0, + "20": 22723148.0, + "21": 22917848.0, + "22": 22944810.0, + "23": 22628814.0, + "24": 22861948.0, + "25": 22636144.0, + "26": 23004472.0, + "27": 22801852.0, + "28": 22999004.0, + "29": 22978964.0, + "30": 22943972.0, + "31": 22908184.0, + "32": 22665242.0, + "33": 22740176.0, + "34": 23076434.0, + "35": 22749892.0, + "36": 22694802.0, + "37": 23104044.0, + "38": 22964560.0, + "39": 22985088.0, + "40": 22750652.0, + "41": 23065016.0, + "42": 22688756.0, + "43": 22987280.0, + "44": 22704512.0, + "45": 22847482.0, + "46": 22733950.0, + "47": 22850374.0, + "48": 22833462.0, + "49": 22889040.0, + "50": 22644900.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.48655, - "3": 0.14271, - "4": 0.13967, - "5": 0.13987, - "6": 0.14039, - "7": 0.14008, - "8": 0.14339, - "9": 0.1387, - "10": 0.14028, - "11": 0.13933, - "12": 0.14022, - "13": 0.14075, - "14": 0.13872, - "15": 0.14042, - "16": 0.14185, - "17": 0.1402, - "18": 0.14062, - "19": 0.13885, - "20": 0.13913, - "21": 0.13948, - "22": 0.13888, - "23": 0.13805, - "24": 0.13946, - "25": 0.14414, - "26": 0.13982, - "27": 0.13865, - "28": 0.14095, - "29": 0.13879, - "30": 0.13921, - "31": 0.14151, - "32": 0.14307, - "33": 0.13838, - "34": 0.1398, - "35": 0.13938, - "36": 0.14126, - "37": 0.14155, - "38": 0.13902, - "39": 0.1409, - "40": 0.14029, - "41": 0.14115, - "42": 0.14031, - "43": 0.14074, - "44": 0.13928, - "45": 0.13869, - "46": 0.13901, - "47": 0.14205, - "48": 0.14117, - "49": 0.14874, - "50": 0.13916 + "2": 6.247, + "3": 0.14876, + "4": 0.14519, + "5": 0.14318, + "6": 0.14486, + "7": 0.14519, + "8": 0.14445, + "9": 0.145, + "10": 0.14487, + "11": 0.14631, + "12": 0.14668, + "13": 0.14343, + "14": 0.14337, + "15": 0.14413, + "16": 0.14564, + "17": 0.15207, + "18": 0.14537, + "19": 0.14551, + "20": 0.14575, + "21": 0.14407, + "22": 0.14478, + "23": 0.14592, + "24": 0.1437, + "25": 0.14361, + "26": 0.14418, + "27": 0.14563, + "28": 0.1447, + "29": 0.14376, + "30": 0.14281, + "31": 0.14404, + "32": 0.14426, + "33": 0.1455, + "34": 0.14214, + "35": 0.14214, + "36": 0.14284, + "37": 0.1423, + "38": 0.14166, + "39": 0.14484, + "40": 0.14123, + "41": 0.14187, + "42": 0.14231, + "43": 0.14633, + "44": 0.1427, + "45": 0.14255, + "46": 0.14262, + "47": 0.14277, + "48": 0.1424, + "49": 0.14275, + "50": 0.14789 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json index 24423fd16b1..a8204639e40 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.91778, - "2": 10.91602, - "3": 10.9198, + "1": 10.91788, + "2": 10.91609, + "3": 10.91977, "4": 10.91999, - "5": 10.91201, - "6": 10.91954, - "7": 10.92034, - "8": 10.91471, - "9": 10.90787, - "10": 10.90661, + "5": 10.91193, + "6": 10.91967, + "7": 10.92039, + "8": 10.91461, + "9": 10.90788, + "10": 10.90662, "11": 10.90292, - "12": 10.90754, - "13": 10.90318, - "14": 10.8975, - "15": 10.86511, - "16": 10.86752, - "17": 10.8629, - "18": 10.85425, - "19": 10.85849, - "20": 10.77184, - "21": 10.76543, - "22": 10.74101, - "23": 10.73028, - "24": 10.71854, - "25": 10.69262, - "26": 10.68889, - "27": 10.63789, - "28": 10.57948, + "12": 10.90752, + "13": 10.90331, + "14": 10.89747, + "15": 10.86508, + "16": 10.86748, + "17": 10.86291, + "18": 10.85426, + "19": 10.85851, + "20": 10.77185, + "21": 10.7654, + "22": 10.741, + "23": 10.73033, + "24": 10.7186, + "25": 10.69268, + "26": 10.68897, + "27": 10.63802, + "28": 10.57955, "29": 10.54148, - "30": 10.50449, - "31": 10.51059, - "32": 10.4944, - "33": 10.4464, - "34": 10.4268, - "35": 10.42715, - "36": 10.39378, - "37": 10.37391, - "38": 10.36201, - "39": 10.33359, - "40": 10.30551, - "41": 10.29099, - "42": 10.25817, - "43": 10.24905, - "44": 10.21682, - "45": 10.227, - "46": 10.17061, - "47": 10.17437, - "48": 10.13475, - "49": 10.13473, - "50": 10.11788 + "30": 10.50454, + "31": 10.51064, + "32": 10.49445, + "33": 10.44646, + "34": 10.42685, + "35": 10.42724, + "36": 10.39377, + "37": 10.37397, + "38": 10.362, + "39": 10.3337, + "40": 10.30562, + "41": 10.29104, + "42": 10.25824, + "43": 10.24913, + "44": 10.21691, + "45": 10.22705, + "46": 10.17066, + "47": 10.17441, + "48": 10.13478, + "49": 10.13485, + "50": 10.11789 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22961698.0, - "2": 22850648.0, - "3": 22708944.0, - "4": 22777238.0, - "5": 22782670.0, - "6": 22745544.0, - "7": 22871068.0, - "8": 22611328.0, - "9": 22758860.0, - "10": 22488844.0, - "11": 22752972.0, - "12": 22641496.0, - "13": 23322216.0, - "14": 22991336.0, - "15": 22717318.0, - "16": 22822488.0, - "17": 22929414.0, - "18": 22994556.0, - "19": 23084612.0, - "20": 22722388.0, - "21": 22918198.0, - "22": 22944376.0, - "23": 22628422.0, - "24": 22862476.0, - "25": 22636572.0, - "26": 23004492.0, - "27": 22801540.0, - "28": 22998012.0, - "29": 22979304.0, - "30": 22944624.0, - "31": 22907846.0, - "32": 22665480.0, - "33": 22739828.0, - "34": 23075384.0, - "35": 22750508.0, - "36": 22694996.0, - "37": 23103238.0, - "38": 22963552.0, - "39": 22984982.0, - "40": 22749680.0, - "41": 23065176.0, - "42": 22688580.0, - "43": 22987652.0, - "44": 22704676.0, - "45": 22847124.0, - "46": 22734064.0, - "47": 22849372.0, - "48": 22832772.0, - "49": 22888554.0, - "50": 22644322.0 + "1": 22961726.0, + "2": 22850560.0, + "3": 22708848.0, + "4": 22777128.0, + "5": 22782776.0, + "6": 22745572.0, + "7": 22871024.0, + "8": 22611400.0, + "9": 22758804.0, + "10": 22488848.0, + "11": 22753076.0, + "12": 22641586.0, + "13": 23322250.0, + "14": 22991348.0, + "15": 22717328.0, + "16": 22822440.0, + "17": 22929468.0, + "18": 22994548.0, + "19": 23084632.0, + "20": 22722448.0, + "21": 22918196.0, + "22": 22944310.0, + "23": 22628442.0, + "24": 22862544.0, + "25": 22636710.0, + "26": 23004344.0, + "27": 22801504.0, + "28": 22997960.0, + "29": 22979424.0, + "30": 22944576.0, + "31": 22907836.0, + "32": 22665460.0, + "33": 22739932.0, + "34": 23075416.0, + "35": 22750576.0, + "36": 22694956.0, + "37": 23103186.0, + "38": 22963456.0, + "39": 22985140.0, + "40": 22749690.0, + "41": 23065180.0, + "42": 22688576.0, + "43": 22987788.0, + "44": 22704648.0, + "45": 22847184.0, + "46": 22734016.0, + "47": 22849236.0, + "48": 22832806.0, + "49": 22888772.0, + "50": 22644324.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.00307, - "3": 2.46501, - "4": 1.71717, - "5": 1.50195, - "6": 2.0154, - "7": 1.88147, - "8": 2.40787, - "9": 2.39934, - "10": 2.18348, - "11": 2.62062, - "12": 1.9888, - "13": 3.01208, - "14": 3.6425, - "15": 4.10603, - "16": 3.39627, - "17": 4.04231, - "18": 3.21431, - "19": 2.64929, - "20": 2.68367, - "21": 3.2317, - "22": 2.82182, - "23": 3.19262, - "24": 2.79362, - "25": 3.71877, - "26": 4.15527, - "27": 2.79149, - "28": 2.64167, - "29": 1.72488, - "30": 2.23461, - "31": 2.14333, - "32": 3.0767, - "33": 1.77825, - "34": 1.7662, - "35": 2.65221, - "36": 2.36333, - "37": 2.34348, - "38": 1.63998, - "39": 2.5093, - "40": 2.17594, - "41": 2.00567, - "42": 2.69642, - "43": 2.00334, - "44": 2.09538, - "45": 2.99263, - "46": 2.10827, - "47": 2.39942, - "48": 2.93432, - "49": 2.07109, - "50": 2.32718 + "2": 6.02919, + "3": 2.18874, + "4": 2.16542, + "5": 1.59884, + "6": 1.38154, + "7": 1.45784, + "8": 2.15836, + "9": 1.86592, + "10": 1.92744, + "11": 2.03649, + "12": 1.73987, + "13": 1.80755, + "14": 2.25243, + "15": 1.48355, + "16": 1.65132, + "17": 2.00593, + "18": 2.05995, + "19": 2.1088, + "20": 1.65313, + "21": 2.15916, + "22": 1.80752, + "23": 1.76357, + "24": 1.58001, + "25": 2.3143, + "26": 2.10306, + "27": 1.80734, + "28": 1.77917, + "29": 1.28319, + "30": 1.78781, + "31": 1.60732, + "32": 3.08685, + "33": 1.58492, + "34": 1.6402, + "35": 2.28672, + "36": 2.31123, + "37": 1.68766, + "38": 1.31698, + "39": 2.10463, + "40": 1.96123, + "41": 2.09881, + "42": 2.16112, + "43": 2.01629, + "44": 1.81023, + "45": 2.41369, + "46": 1.77693, + "47": 1.88503, + "48": 2.30102, + "49": 1.58618, + "50": 1.96197 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json index bc10a7b3abd..c4829cf3323 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92679, - "2": 10.92501, - "3": 10.92521, - "4": 10.92422, - "5": 10.92547, - "6": 10.91033, - "7": 10.91228, - "8": 10.9229, - "9": 10.92921, - "10": 10.93096, - "11": 10.92107, + "1": 10.92677, + "2": 10.92498, + "3": 10.9252, + "4": 10.92429, + "5": 10.92541, + "6": 10.91034, + "7": 10.91232, + "8": 10.92285, + "9": 10.92916, + "10": 10.93091, + "11": 10.92106, "12": 10.9162, - "13": 10.91442, - "14": 10.91553, - "15": 10.91878, - "16": 10.90576, - "17": 10.91144, - "18": 10.89352, - "19": 10.9045, - "20": 10.88089, - "21": 10.86586, + "13": 10.91435, + "14": 10.91551, + "15": 10.91882, + "16": 10.90582, + "17": 10.91136, + "18": 10.89359, + "19": 10.90454, + "20": 10.8809, + "21": 10.86585, "22": 10.85398, - "23": 10.85622, - "24": 10.84644, - "25": 10.8478, - "26": 10.84328, + "23": 10.85608, + "24": 10.84649, + "25": 10.84777, + "26": 10.84327, "27": 10.83811, - "28": 10.76378, - "29": 10.75291, - "30": 10.74291, - "31": 10.71651, - "32": 10.7293, - "33": 10.70833, - "34": 10.67987, - "35": 10.67977, - "36": 10.67754, - "37": 10.64937, - "38": 10.62141, - "39": 10.58145, - "40": 10.5672, - "41": 10.5547, - "42": 10.52441, - "43": 10.5169, - "44": 10.49635, - "45": 10.50699, - "46": 10.46295, - "47": 10.464, - "48": 10.42072, - "49": 10.40804, - "50": 10.39052 + "28": 10.76387, + "29": 10.75296, + "30": 10.74298, + "31": 10.71659, + "32": 10.72937, + "33": 10.70832, + "34": 10.67995, + "35": 10.67987, + "36": 10.67766, + "37": 10.64938, + "38": 10.62144, + "39": 10.58151, + "40": 10.56728, + "41": 10.55472, + "42": 10.52448, + "43": 10.51695, + "44": 10.49638, + "45": 10.50711, + "46": 10.46296, + "47": 10.46401, + "48": 10.42075, + "49": 10.40821, + "50": 10.39062 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22961676.0, - "2": 22849608.0, - "3": 22708396.0, - "4": 22776906.0, - "5": 22783086.0, - "6": 22744204.0, - "7": 22870900.0, - "8": 22610356.0, - "9": 22758672.0, - "10": 22488532.0, - "11": 22754076.0, - "12": 22641504.0, - "13": 23321644.0, - "14": 22991400.0, - "15": 22717964.0, - "16": 22822160.0, - "17": 22928174.0, - "18": 22993726.0, - "19": 23083696.0, - "20": 22722568.0, - "21": 22916920.0, - "22": 22944496.0, - "23": 22627968.0, - "24": 22861256.0, - "25": 22635224.0, + "1": 22961644.0, + "2": 22849640.0, + "3": 22708490.0, + "4": 22776876.0, + "5": 22783124.0, + "6": 22744210.0, + "7": 22870980.0, + "8": 22610342.0, + "9": 22758686.0, + "10": 22488428.0, + "11": 22754040.0, + "12": 22641422.0, + "13": 23321638.0, + "14": 22991496.0, + "15": 22717886.0, + "16": 22822172.0, + "17": 22928028.0, + "18": 22993656.0, + "19": 23083736.0, + "20": 22722640.0, + "21": 22916940.0, + "22": 22944400.0, + "23": 22627992.0, + "24": 22861248.0, + "25": 22635202.0, "26": 23004420.0, - "27": 22800372.0, - "28": 22998250.0, - "29": 22977944.0, - "30": 22943312.0, - "31": 22906826.0, - "32": 22665452.0, - "33": 22740024.0, - "34": 23074180.0, - "35": 22749940.0, - "36": 22694928.0, - "37": 23102576.0, - "38": 22963904.0, - "39": 22983708.0, - "40": 22748514.0, - "41": 23064272.0, - "42": 22687828.0, - "43": 22986744.0, - "44": 22703500.0, - "45": 22845972.0, - "46": 22732688.0, - "47": 22848320.0, - "48": 22831952.0, - "49": 22887420.0, - "50": 22643594.0 + "27": 22800376.0, + "28": 22998380.0, + "29": 22977832.0, + "30": 22943328.0, + "31": 22906864.0, + "32": 22665466.0, + "33": 22740194.0, + "34": 23074260.0, + "35": 22749998.0, + "36": 22694964.0, + "37": 23102570.0, + "38": 22963848.0, + "39": 22983736.0, + "40": 22748524.0, + "41": 23064284.0, + "42": 22687892.0, + "43": 22986764.0, + "44": 22703484.0, + "45": 22845986.0, + "46": 22732752.0, + "47": 22848242.0, + "48": 22832138.0, + "49": 22887472.0, + "50": 22643654.0 } }, "mem-allocated-bytes": { @@ -179,33 +179,33 @@ "2": 2840251904.0, "3": 2840251904.0, "4": 2840251904.0, - "5": 2841169408.0, - "6": 2841169408.0, - "7": 2841169408.0, - "8": 2841169408.0, - "9": 2841169408.0, - "10": 2841169408.0, - "11": 2841169408.0, - "12": 2841169408.0, - "13": 2841169408.0, - "14": 2841169408.0, - "15": 2841169408.0, - "16": 2841169408.0, - "17": 2841169408.0, - "18": 2841169408.0, - "19": 2841169408.0, - "20": 2841169408.0, - "21": 2841169408.0, - "22": 2841169408.0, - "23": 2841169408.0, - "24": 2841169408.0, - "25": 2841169408.0, - "26": 2841169408.0, - "27": 2841169408.0, - "28": 2841169408.0, - "29": 2841169408.0, - "30": 2841169408.0, - "31": 2841169408.0, + "5": 2840251904.0, + "6": 2840251904.0, + "7": 2840251904.0, + "8": 2840251904.0, + "9": 2840251904.0, + "10": 2840251904.0, + "11": 2840251904.0, + "12": 2840251904.0, + "13": 2840251904.0, + "14": 2840251904.0, + "15": 2840251904.0, + "16": 2840251904.0, + "17": 2840251904.0, + "18": 2840251904.0, + "19": 2840251904.0, + "20": 2840251904.0, + "21": 2840251904.0, + "22": 2840251904.0, + "23": 2840251904.0, + "24": 2840251904.0, + "25": 2840251904.0, + "26": 2840251904.0, + "27": 2840251904.0, + "28": 2840251904.0, + "29": 2840251904.0, + "30": 2840251904.0, + "31": 2840251904.0, "32": 2841169408.0, "33": 2841169408.0, "34": 2841169408.0, @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.85847, - "3": 0.07317, - "4": 0.07284, - "5": 0.07195, - "6": 0.07422, - "7": 0.07283, - "8": 0.07174, - "9": 0.07266, - "10": 0.0717, - "11": 0.07102, - "12": 0.07181, - "13": 0.07157, - "14": 0.07022, - "15": 0.07232, - "16": 0.07228, - "17": 0.07172, - "18": 0.07174, - "19": 0.07099, - "20": 0.07215, - "21": 0.07285, - "22": 0.07143, - "23": 0.0719, - "24": 0.07066, - "25": 0.0709, - "26": 0.07287, - "27": 0.07095, - "28": 0.07074, - "29": 0.07198, - "30": 0.07139, - "31": 0.07124, - "32": 0.07122, - "33": 0.07196, - "34": 0.07084, - "35": 0.07457, - "36": 0.07092, - "37": 0.07103, - "38": 0.07077, - "39": 0.07044, - "40": 0.07129, - "41": 0.07181, - "42": 0.07137, - "43": 0.07124, - "44": 0.07128, - "45": 0.07014, - "46": 0.07083, - "47": 0.07142, - "48": 0.07092, - "49": 0.07082, - "50": 0.0712 + "2": 7.5188, + "3": 0.07341, + "4": 0.07346, + "5": 0.07295, + "6": 0.07322, + "7": 0.07377, + "8": 0.07615, + "9": 0.0734, + "10": 0.07762, + "11": 0.07613, + "12": 0.07447, + "13": 0.07241, + "14": 0.07119, + "15": 0.07199, + "16": 0.07578, + "17": 0.07455, + "18": 0.07183, + "19": 0.07398, + "20": 0.07439, + "21": 0.0756, + "22": 0.07417, + "23": 0.07399, + "24": 0.07292, + "25": 0.07626, + "26": 0.07556, + "27": 0.07573, + "28": 0.0729, + "29": 0.07267, + "30": 0.07423, + "31": 0.07389, + "32": 0.07413, + "33": 0.07593, + "34": 0.07418, + "35": 0.07373, + "36": 0.07239, + "37": 0.07332, + "38": 0.07692, + "39": 0.07443, + "40": 0.07287, + "41": 0.07845, + "42": 0.07404, + "43": 0.07276, + "44": 0.07625, + "45": 0.07491, + "46": 0.07542, + "47": 0.07424, + "48": 0.07312, + "49": 0.0746, + "50": 0.07427 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json index 8ce195c0ecd..dd12fa45f78 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.90673, - "2": 10.91263, - "3": 10.91376, - "4": 10.90507, - "5": 10.90666, - "6": 10.9031, - "7": 10.9059, - "8": 10.91387, - "9": 10.90878, - "10": 10.9237, - "11": 10.90423, + "1": 10.90666, + "2": 10.91256, + "3": 10.91382, + "4": 10.90512, + "5": 10.90674, + "6": 10.90308, + "7": 10.90577, + "8": 10.91388, + "9": 10.90873, + "10": 10.92368, + "11": 10.9042, "12": 10.90022, - "13": 10.91206, - "14": 10.91487, - "15": 10.89545, - "16": 10.88862, - "17": 10.89879, - "18": 10.88901, + "13": 10.91207, + "14": 10.91489, + "15": 10.8954, + "16": 10.88861, + "17": 10.89883, + "18": 10.88906, "19": 10.89342, - "20": 10.85175, + "20": 10.85169, "21": 10.85495, - "22": 10.85415, - "23": 10.84295, - "24": 10.82607, - "25": 10.82944, - "26": 10.83505, - "27": 10.81605, - "28": 10.76187, - "29": 10.73276, - "30": 10.71482, - "31": 10.70796, - "32": 10.71541, - "33": 10.683, - "34": 10.6638, - "35": 10.66095, - "36": 10.64805, - "37": 10.62756, - "38": 10.60871, - "39": 10.55474, - "40": 10.54268, - "41": 10.52944, - "42": 10.51104, - "43": 10.50087, - "44": 10.46797, - "45": 10.47461, - "46": 10.4319, - "47": 10.42659, - "48": 10.39161, - "49": 10.3871, - "50": 10.36849 + "22": 10.85414, + "23": 10.84293, + "24": 10.8261, + "25": 10.82939, + "26": 10.83508, + "27": 10.8161, + "28": 10.76185, + "29": 10.73282, + "30": 10.7148, + "31": 10.70789, + "32": 10.71547, + "33": 10.68298, + "34": 10.66386, + "35": 10.66104, + "36": 10.64814, + "37": 10.62766, + "38": 10.60876, + "39": 10.55484, + "40": 10.54274, + "41": 10.52951, + "42": 10.51109, + "43": 10.50092, + "44": 10.46807, + "45": 10.47463, + "46": 10.43196, + "47": 10.4267, + "48": 10.39165, + "49": 10.38717, + "50": 10.36854 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 22962016.0, - "2": 22848584.0, - "3": 22708364.0, - "4": 22777148.0, - "5": 22782800.0, - "6": 22745854.0, - "7": 22870548.0, - "8": 22610756.0, - "9": 22757820.0, - "10": 22489088.0, - "11": 22753460.0, - "12": 22640528.0, - "13": 23321632.0, - "14": 22990940.0, - "15": 22717336.0, - "16": 22821172.0, - "17": 22928714.0, - "18": 22994028.0, - "19": 23084446.0, - "20": 22722252.0, - "21": 22917710.0, - "22": 22945000.0, - "23": 22627552.0, - "24": 22861514.0, + "1": 22962108.0, + "2": 22848672.0, + "3": 22708304.0, + "4": 22777134.0, + "5": 22782820.0, + "6": 22745902.0, + "7": 22870540.0, + "8": 22610754.0, + "9": 22757862.0, + "10": 22489126.0, + "11": 22753440.0, + "12": 22640536.0, + "13": 23321504.0, + "14": 22990882.0, + "15": 22717276.0, + "16": 22821292.0, + "17": 22928660.0, + "18": 22994056.0, + "19": 23084428.0, + "20": 22722272.0, + "21": 22917716.0, + "22": 22944964.0, + "23": 22627636.0, + "24": 22861476.0, "25": 22635880.0, - "26": 23004104.0, - "27": 22801224.0, - "28": 22998068.0, - "29": 22978356.0, - "30": 22943064.0, - "31": 22906302.0, - "32": 22664320.0, - "33": 22739984.0, - "34": 23075192.0, - "35": 22749556.0, - "36": 22693736.0, - "37": 23103292.0, - "38": 22964034.0, - "39": 22983724.0, - "40": 22749088.0, - "41": 23063492.0, - "42": 22687832.0, - "43": 22986674.0, - "44": 22703486.0, - "45": 22846044.0, - "46": 22732872.0, - "47": 22848216.0, - "48": 22832092.0, - "49": 22887284.0, - "50": 22642954.0 + "26": 23004070.0, + "27": 22801236.0, + "28": 22998040.0, + "29": 22978500.0, + "30": 22943076.0, + "31": 22906274.0, + "32": 22664220.0, + "33": 22739872.0, + "34": 23075182.0, + "35": 22749520.0, + "36": 22693664.0, + "37": 23103284.0, + "38": 22963964.0, + "39": 22983794.0, + "40": 22749016.0, + "41": 23063564.0, + "42": 22687764.0, + "43": 22986694.0, + "44": 22703464.0, + "45": 22846068.0, + "46": 22732996.0, + "47": 22848258.0, + "48": 22832112.0, + "49": 22887248.0, + "50": 22642880.0 } }, "mem-allocated-bytes": { @@ -176,43 +176,43 @@ "step_interval": 1, "values": { "1": 2610027008.0, - "2": 2841171456.0, + "2": 2841169408.0, "3": 2841171456.0, "4": 2841171456.0, "5": 2841171456.0, "6": 2841171456.0, "7": 2841171456.0, - "8": 2841171456.0, - "9": 2841171456.0, - "10": 2841171456.0, - "11": 2841171456.0, - "12": 2841171456.0, - "13": 2841171456.0, - "14": 2841171456.0, - "15": 2841171456.0, - "16": 2841171456.0, - "17": 2841171456.0, - "18": 2841171456.0, - "19": 2841171456.0, - "20": 2841171456.0, - "21": 2841171456.0, - "22": 2841171456.0, - "23": 2841171456.0, - "24": 2841171456.0, - "25": 2841171456.0, - "26": 2841171456.0, - "27": 2841171456.0, - "28": 2841171456.0, - "29": 2841171456.0, - "30": 2841171456.0, - "31": 2841171456.0, - "32": 2841171456.0, - "33": 2841171456.0, - "34": 2841171456.0, - "35": 2841171456.0, - "36": 2841171456.0, - "37": 2841171456.0, - "38": 2841171456.0, + "8": 2841826816.0, + "9": 2841826816.0, + "10": 2841826816.0, + "11": 2841826816.0, + "12": 2841826816.0, + "13": 2841826816.0, + "14": 2841826816.0, + "15": 2842088960.0, + "16": 2842088960.0, + "17": 2842088960.0, + "18": 2842088960.0, + "19": 2842088960.0, + "20": 2842088960.0, + "21": 2842088960.0, + "22": 2842088960.0, + "23": 2842088960.0, + "24": 2842088960.0, + "25": 2842088960.0, + "26": 2842088960.0, + "27": 2842088960.0, + "28": 2842088960.0, + "29": 2842088960.0, + "30": 2842088960.0, + "31": 2842088960.0, + "32": 2842088960.0, + "33": 2842088960.0, + "34": 2842088960.0, + "35": 2842088960.0, + "36": 2842088960.0, + "37": 2842088960.0, + "38": 2842088960.0, "39": 2842088960.0, "40": 2842088960.0, "41": 2842088960.0, @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.9373, - "3": 1.87779, - "4": 1.77285, - "5": 1.60706, - "6": 1.73856, - "7": 1.71512, - "8": 2.3102, - "9": 2.34207, - "10": 1.712, - "11": 2.04949, - "12": 1.51573, - "13": 1.96266, - "14": 1.61171, - "15": 1.56691, - "16": 1.85963, - "17": 2.43498, - "18": 1.90961, - "19": 1.75855, - "20": 1.86545, - "21": 2.07982, - "22": 1.38767, - "23": 1.67867, - "24": 1.60979, - "25": 2.12194, - "26": 2.38425, - "27": 1.70776, - "28": 1.8058, - "29": 1.63302, - "30": 1.59172, - "31": 1.4531, - "32": 2.25714, - "33": 1.57842, - "34": 1.57638, - "35": 2.03622, - "36": 2.02296, - "37": 1.65123, - "38": 1.47971, - "39": 1.94619, - "40": 1.46247, - "41": 1.79716, - "42": 2.3247, - "43": 1.63639, - "44": 1.53309, - "45": 1.96158, - "46": 1.87269, - "47": 1.81212, - "48": 2.09298, - "49": 1.44854, - "50": 1.77975 + "2": 6.53977, + "3": 2.16799, + "4": 2.05262, + "5": 1.67891, + "6": 1.56076, + "7": 1.38927, + "8": 1.91945, + "9": 2.47565, + "10": 1.84838, + "11": 1.8321, + "12": 1.63712, + "13": 2.00428, + "14": 1.74267, + "15": 1.8807, + "16": 1.67898, + "17": 2.39913, + "18": 1.5976, + "19": 1.96188, + "20": 1.99695, + "21": 1.87049, + "22": 1.6075, + "23": 2.30566, + "24": 1.6751, + "25": 2.30101, + "26": 2.64019, + "27": 2.40278, + "28": 2.2643, + "29": 1.83472, + "30": 2.2083, + "31": 1.63448, + "32": 2.71684, + "33": 1.61254, + "34": 1.9108, + "35": 1.99889, + "36": 2.09762, + "37": 2.34162, + "38": 1.7158, + "39": 1.92944, + "40": 1.9202, + "41": 1.73484, + "42": 2.32617, + "43": 1.92165, + "44": 2.12406, + "45": 2.68292, + "46": 1.68484, + "47": 1.84373, + "48": 2.24371, + "49": 1.60563, + "50": 2.22446 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json index 4c91171b41f..4669e21299e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93132, - "2": 10.92661, - "3": 10.92492, - "4": 10.92081, - "5": 10.91909, - "6": 10.91583, - "7": 10.92398, - "8": 10.91668, - "9": 10.91511, - "10": 10.91953, + "2": 10.92668, + "3": 10.92486, + "4": 10.92085, + "5": 10.91905, + "6": 10.91578, + "7": 10.92394, + "8": 10.9167, + "9": 10.91517, + "10": 10.91951, "11": 10.91089, - "12": 10.90468, - "13": 10.89957, - "14": 10.9008, + "12": 10.90466, + "13": 10.89953, + "14": 10.90081, "15": 10.88316, - "16": 10.86332, - "17": 10.8821, - "18": 10.86576, - "19": 10.86883, - "20": 10.79012, - "21": 10.78745, - "22": 10.77483, - "23": 10.76341, - "24": 10.73372, - "25": 10.73326, - "26": 10.71699, - "27": 10.67915, - "28": 10.61678, - "29": 10.59025, - "30": 10.55953, - "31": 10.56388, - "32": 10.54876, - "33": 10.513, - "34": 10.48531, - "35": 10.48189, - "36": 10.46061, - "37": 10.43614, - "38": 10.43665, - "39": 10.39487, - "40": 10.3885, - "41": 10.36649, - "42": 10.34951, - "43": 10.32455, - "44": 10.30276, - "45": 10.30966, - "46": 10.27275, - "47": 10.25872, - "48": 10.2129, - "49": 10.21685, - "50": 10.21012, - "51": 10.21222, + "16": 10.86335, + "17": 10.88219, + "18": 10.86584, + "19": 10.86886, + "20": 10.79014, + "21": 10.78744, + "22": 10.77485, + "23": 10.76339, + "24": 10.73368, + "25": 10.73329, + "26": 10.71706, + "27": 10.67916, + "28": 10.61679, + "29": 10.59026, + "30": 10.55955, + "31": 10.56383, + "32": 10.54878, + "33": 10.51301, + "34": 10.48532, + "35": 10.48199, + "36": 10.46066, + "37": 10.43613, + "38": 10.43677, + "39": 10.3949, + "40": 10.38854, + "41": 10.36664, + "42": 10.34957, + "43": 10.32461, + "44": 10.30278, + "45": 10.3097, + "46": 10.2727, + "47": 10.25874, + "48": 10.213, + "49": 10.21691, + "50": 10.21015, + "51": 10.21226, "52": 10.16487, - "53": 10.17139, - "54": 10.13369, - "55": 10.10843, - "56": 10.13281, - "57": 10.11978, - "58": 10.12937, - "59": 10.07403, - "60": 10.09748, - "61": 10.05119, - "62": 10.01666, - "63": 10.0873, - "64": 10.03655, - "65": 10.0096, - "66": 10.04751, - "67": 10.02252, - "68": 9.98732, - "69": 10.00136, - "70": 9.98631, + "53": 10.17133, + "54": 10.13377, + "55": 10.10845, + "56": 10.13289, + "57": 10.11983, + "58": 10.12943, + "59": 10.07413, + "60": 10.09744, + "61": 10.0512, + "62": 10.01672, + "63": 10.08737, + "64": 10.03661, + "65": 10.00964, + "66": 10.04755, + "67": 10.02257, + "68": 9.98735, + "69": 10.00135, + "70": 9.98634, "71": 10.00761, - "72": 9.99372, - "73": 9.98419, - "74": 9.97133, - "75": 9.92977, - "76": 9.96826, - "77": 9.97256, - "78": 9.92117, - "79": 9.91774, - "80": 9.93559, - "81": 9.95704, + "72": 9.99378, + "73": 9.98423, + "74": 9.97138, + "75": 9.92982, + "76": 9.96831, + "77": 9.9726, + "78": 9.9212, + "79": 9.91778, + "80": 9.93564, + "81": 9.95713, "82": 9.8975, - "83": 9.86594, - "84": 9.80667, - "85": 9.78948, - "86": 9.89884, - "87": 9.9096, - "88": 9.89143, - "89": 9.8332, - "90": 9.82672, - "91": 9.83686, - "92": 9.82593, - "93": 9.76052, - "94": 9.83258, - "95": 9.82135, - "96": 9.80912, - "97": 9.75581, - "98": 9.78097, - "99": 9.82255, - "100": 9.71473 + "83": 9.86599, + "84": 9.8067, + "85": 9.78953, + "86": 9.89891, + "87": 9.90966, + "88": 9.89147, + "89": 9.83321, + "90": 9.82668, + "91": 9.83691, + "92": 9.82594, + "93": 9.76057, + "94": 9.8326, + "95": 9.82136, + "96": 9.80918, + "97": 9.7559, + "98": 9.78098, + "99": 9.82253, + "100": 9.71476 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1749.0, - "2": 1763.0, - "3": 1797.0, - "4": 1864.0, - "5": 1778.0, - "6": 1808.0, - "7": 1884.0, - "8": 1743.0, - "9": 1830.0, - "10": 1705.0, - "11": 1709.0, - "12": 1746.0, - "13": 1889.0, - "14": 1891.0, - "15": 1673.0, - "16": 1804.0, - "17": 1890.0, - "18": 1765.0, - "19": 1808.0, - "20": 1751.0, - "21": 1791.0, - "22": 1829.0, - "23": 1754.0, - "24": 1896.0, - "25": 1875.0, - "26": 1746.0, - "27": 1927.0, - "28": 1870.0, - "29": 1917.0, - "30": 1958.0, - "31": 2048.0, - "32": 1956.0, - "33": 1960.0, - "34": 2127.0, - "35": 2143.0, - "36": 2176.0, - "37": 2256.0, - "38": 2174.0, - "39": 2222.0, - "40": 2340.0, - "41": 2340.0, - "42": 2098.0, - "43": 2319.0, - "44": 2276.0, - "45": 2578.0, - "46": 2359.0, - "47": 2486.0, - "48": 2651.0, - "49": 2703.0, - "50": 2667.0, - "51": 2545.0, - "52": 2693.0, - "53": 2674.0, - "54": 2864.0, - "55": 2548.0, - "56": 2687.0, - "57": 2267.0, - "58": 3482.0, - "59": 2868.0, - "60": 2947.0, - "61": 2753.0, - "62": 3072.0, - "63": 3159.0, - "64": 3606.0, - "65": 2787.0, - "66": 2953.0, - "67": 3939.0, - "68": 3417.0, - "69": 2939.0, - "70": 3304.0, - "71": 3249.0, - "72": 3112.0, - "73": 3534.0, - "74": 3391.0, - "75": 3282.0, - "76": 3365.0, - "77": 3679.0, - "78": 3386.0, - "79": 3272.0, - "80": 3082.0, - "81": 3299.0, - "82": 3275.0, - "83": 3155.0, - "84": 3100.0, - "85": 2855.0, - "86": 3078.0, - "87": 3022.0, - "88": 3199.0, - "89": 3214.0, - "90": 4149.0, - "91": 2826.0, - "92": 3119.0, - "93": 3109.0, - "94": 3441.0, - "95": 3281.0, - "96": 3811.0, - "97": 3497.0, - "98": 3532.0, - "99": 3477.0, - "100": 3341.0 + "1": 1816.0, + "2": 1784.0, + "3": 1798.0, + "4": 1787.0, + "5": 1838.0, + "6": 1730.0, + "7": 2008.0, + "8": 1738.0, + "9": 1867.0, + "10": 1799.0, + "11": 1780.0, + "12": 1779.0, + "13": 1801.0, + "14": 1917.0, + "15": 1738.0, + "16": 1784.0, + "17": 1906.0, + "18": 1803.0, + "19": 1758.0, + "20": 1700.0, + "21": 1802.0, + "22": 1799.0, + "23": 1809.0, + "24": 1850.0, + "25": 1759.0, + "26": 1863.0, + "27": 1833.0, + "28": 1879.0, + "29": 2024.0, + "30": 1899.0, + "31": 1984.0, + "32": 2023.0, + "33": 2047.0, + "34": 2105.0, + "35": 2153.0, + "36": 2130.0, + "37": 2360.0, + "38": 2140.0, + "39": 2199.0, + "40": 2311.0, + "41": 2276.0, + "42": 2157.0, + "43": 2376.0, + "44": 2340.0, + "45": 2400.0, + "46": 2392.0, + "47": 2438.0, + "48": 2564.0, + "49": 2759.0, + "50": 2495.0, + "51": 2616.0, + "52": 2712.0, + "53": 2706.0, + "54": 2801.0, + "55": 2533.0, + "56": 2621.0, + "57": 2361.0, + "58": 3597.0, + "59": 2884.0, + "60": 2897.0, + "61": 2687.0, + "62": 3162.0, + "63": 3182.0, + "64": 3681.0, + "65": 2674.0, + "66": 3051.0, + "67": 3774.0, + "68": 3460.0, + "69": 2899.0, + "70": 3200.0, + "71": 3252.0, + "72": 3072.0, + "73": 3420.0, + "74": 3350.0, + "75": 3050.0, + "76": 3361.0, + "77": 3664.0, + "78": 3297.0, + "79": 3337.0, + "80": 3041.0, + "81": 3379.0, + "82": 3299.0, + "83": 3074.0, + "84": 3084.0, + "85": 2867.0, + "86": 3009.0, + "87": 2993.0, + "88": 3244.0, + "89": 3110.0, + "90": 4119.0, + "91": 3004.0, + "92": 3181.0, + "93": 3145.0, + "94": 3288.0, + "95": 3206.0, + "96": 3791.0, + "97": 3560.0, + "98": 3354.0, + "99": 3376.0, + "100": 3403.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.87058, - "3": 0.14701, - "4": 1.74329, - "5": 0.39339, - "6": 1.04265, - "7": 1.24208, - "8": 1.84418, - "9": 1.34809, - "10": 0.4529, - "11": 1.97844, - "12": 0.50437, - "13": 1.04936, - "14": 1.5431, - "15": 0.77848, - "16": 1.74381, - "17": 1.06464, - "18": 1.43904, - "19": 0.62659, - "20": 0.82054, - "21": 1.16783, - "22": 0.74085, - "23": 1.60571, - "24": 0.40254, - "25": 1.61209, - "26": 0.80727, - "27": 1.12032, - "28": 0.80779, - "29": 1.21616, - "30": 0.15192, - "31": 0.38017, - "32": 1.4174, - "33": 1.28283, - "34": 0.8669, - "35": 1.12703, - "36": 0.93849, - "37": 1.26457, - "38": 0.85287, - "39": 1.21165, - "40": 1.37614, - "41": 1.07684, - "42": 0.83559, - "43": 0.6735, - "44": 1.49902, - "45": 0.86528, - "46": 0.68184, - "47": 1.0296, - "48": 1.16521, - "49": 0.96442, - "50": 0.37884, - "51": 0.18367, - "52": 0.20666, - "53": 1.26983, - "54": 1.38124, - "55": 1.42823, - "56": 1.63453, - "57": 1.10568, - "58": 0.79131, - "59": 1.1396, - "60": 1.02703, - "61": 1.37674, - "62": 1.6992, - "63": 0.38795, - "64": 1.27667, - "65": 0.99906, - "66": 1.46019, - "67": 1.52888, - "68": 0.41083, - "69": 1.09554, - "70": 1.27494, - "71": 1.02772, - "72": 0.98292, - "73": 0.50204, - "74": 0.8382, - "75": 1.09369, - "76": 0.84162, - "77": 0.83643, - "78": 1.29648, - "79": 1.04829, - "80": 1.36637, - "81": 1.18688, - "82": 1.09685, - "83": 0.80535, - "84": 0.8312, - "85": 0.96307, - "86": 1.2134, - "87": 1.48783, - "88": 1.06343, - "89": 0.68606, - "90": 1.37786, - "91": 0.71266, - "92": 1.61009, - "93": 0.20605, - "94": 0.98568, - "95": 1.61806, - "96": 1.18797, - "97": 0.52742, - "98": 0.47026, - "99": 2.18459, - "100": 0.44562 + "2": 5.10037, + "3": 0.16273, + "4": 0.16456, + "5": 0.16919, + "6": 0.16152, + "7": 0.16088, + "8": 0.1609, + "9": 0.16502, + "10": 0.16075, + "11": 0.16287, + "12": 0.16284, + "13": 0.16408, + "14": 0.16127, + "15": 0.16304, + "16": 0.16091, + "17": 0.16381, + "18": 0.1618, + "19": 0.16283, + "20": 0.16242, + "21": 0.1614, + "22": 0.1607, + "23": 0.16177, + "24": 0.1666, + "25": 0.16212, + "26": 0.16161, + "27": 0.1613, + "28": 0.16023, + "29": 0.16379, + "30": 0.16091, + "31": 0.16076, + "32": 0.16242, + "33": 0.1616, + "34": 0.16058, + "35": 0.16053, + "36": 0.16129, + "37": 0.16522, + "38": 0.16125, + "39": 0.1631, + "40": 0.16214, + "41": 0.16153, + "42": 0.1646, + "43": 0.16775, + "44": 0.16912, + "45": 0.16417, + "46": 0.16125, + "47": 0.16859, + "48": 0.16688, + "49": 0.16971, + "50": 0.16658, + "51": 0.18782, + "52": 0.22741, + "53": 0.17276, + "54": 0.1709, + "55": 0.16581, + "56": 0.16467, + "57": 0.16822, + "58": 0.1658, + "59": 0.17612, + "60": 0.17449, + "61": 0.16832, + "62": 0.16107, + "63": 0.16112, + "64": 0.17446, + "65": 0.17654, + "66": 0.16586, + "67": 0.17019, + "68": 0.16849, + "69": 0.16659, + "70": 0.1669, + "71": 0.17667, + "72": 0.16764, + "73": 0.17529, + "74": 0.1747, + "75": 0.171, + "76": 0.16845, + "77": 0.16857, + "78": 0.1714, + "79": 0.16331, + "80": 0.16306, + "81": 0.16171, + "82": 0.16094, + "83": 0.16207, + "84": 0.16282, + "85": 0.16111, + "86": 0.16141, + "87": 0.15999, + "88": 0.15967, + "89": 0.16365, + "90": 0.16659, + "91": 0.16497, + "92": 0.15996, + "93": 0.16216, + "94": 0.15949, + "95": 0.16267, + "96": 0.16154, + "97": 0.16174, + "98": 0.16413, + "99": 0.16328, + "100": 0.16469 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json index b695b1b0405..89f2d1fb9bd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95267, - "2": 10.9487, + "1": 10.95268, + "2": 10.94868, "3": 10.95574, - "4": 10.94431, - "5": 10.94704, + "4": 10.94432, + "5": 10.94698, "6": 10.94549, - "7": 10.95237, - "8": 10.93777, - "9": 10.94948, - "10": 10.93746, - "11": 10.93723, - "12": 10.93039, - "13": 10.9262, - "14": 10.92334, - "15": 10.90308, - "16": 10.8917, - "17": 10.90057, - "18": 10.88579, - "19": 10.88639, - "20": 10.8026, - "21": 10.79321, - "22": 10.78322, - "23": 10.77587, - "24": 10.73896, - "25": 10.75417, - "26": 10.72426, - "27": 10.68676, - "28": 10.60946, - "29": 10.58265, - "30": 10.57715, - "31": 10.56009, - "32": 10.54731, - "33": 10.51567, - "34": 10.47312, - "35": 10.47454, - "36": 10.46305, - "37": 10.41894, - "38": 10.43058, - "39": 10.39286, - "40": 10.37587, - "41": 10.35957, - "42": 10.34601, - "43": 10.30948, - "44": 10.28618, - "45": 10.29673, - "46": 10.26334, - "47": 10.24621, - "48": 10.20438, - "49": 10.19586, - "50": 10.1985, - "51": 10.20538, - "52": 10.15528, - "53": 10.15622, - "54": 10.12185, - "55": 10.09957, - "56": 10.12109, - "57": 10.10642, - "58": 10.11963, - "59": 10.06902, - "60": 10.0833, - "61": 10.03794, - "62": 10.01121, - "63": 10.07814, - "64": 10.04133, - "65": 10.01344, - "66": 10.03614, - "67": 10.01253, - "68": 9.97682, - "69": 9.99821, + "7": 10.95231, + "8": 10.93786, + "9": 10.94945, + "10": 10.93747, + "11": 10.93726, + "12": 10.93046, + "13": 10.92612, + "14": 10.9233, + "15": 10.90304, + "16": 10.89173, + "17": 10.90056, + "18": 10.88581, + "19": 10.88646, + "20": 10.80252, + "21": 10.79332, + "22": 10.78326, + "23": 10.776, + "24": 10.73889, + "25": 10.7542, + "26": 10.72434, + "27": 10.68683, + "28": 10.60948, + "29": 10.58272, + "30": 10.57709, + "31": 10.56013, + "32": 10.54736, + "33": 10.5158, + "34": 10.47315, + "35": 10.4746, + "36": 10.46311, + "37": 10.41904, + "38": 10.4306, + "39": 10.39293, + "40": 10.37584, + "41": 10.3596, + "42": 10.34607, + "43": 10.30958, + "44": 10.28627, + "45": 10.29677, + "46": 10.26336, + "47": 10.24627, + "48": 10.20441, + "49": 10.19588, + "50": 10.19855, + "51": 10.20543, + "52": 10.15539, + "53": 10.15623, + "54": 10.12189, + "55": 10.09963, + "56": 10.12111, + "57": 10.10654, + "58": 10.11962, + "59": 10.06906, + "60": 10.08331, + "61": 10.03797, + "62": 10.01126, + "63": 10.07817, + "64": 10.04135, + "65": 10.01357, + "66": 10.03619, + "67": 10.01264, + "68": 9.97681, + "69": 9.99825, "70": 9.98055, - "71": 10.00729, - "72": 9.98249, - "73": 9.97937, - "74": 9.96109, - "75": 9.93579, - "76": 9.96744, + "71": 10.00732, + "72": 9.98252, + "73": 9.97936, + "74": 9.96117, + "75": 9.93586, + "76": 9.9674, "77": 9.96184, - "78": 9.90936, - "79": 9.9194, - "80": 9.9384, - "81": 9.96274, - "82": 9.89505, - "83": 9.85891, - "84": 9.7885, - "85": 9.7864, - "86": 9.88552, - "87": 9.91121, - "88": 9.88699, - "89": 9.8184, - "90": 9.81194, - "91": 9.83103, - "92": 9.81784, - "93": 9.74871, - "94": 9.83439, - "95": 9.82929, - "96": 9.80802, - "97": 9.74284, - "98": 9.78138, - "99": 9.82283, - "100": 9.71499 + "78": 9.90945, + "79": 9.91948, + "80": 9.93845, + "81": 9.96282, + "82": 9.89511, + "83": 9.85893, + "84": 9.78855, + "85": 9.78647, + "86": 9.8855, + "87": 9.91123, + "88": 9.88706, + "89": 9.81846, + "90": 9.81195, + "91": 9.8311, + "92": 9.81789, + "93": 9.74874, + "94": 9.8344, + "95": 9.82931, + "96": 9.80803, + "97": 9.74294, + "98": 9.78146, + "99": 9.82289, + "100": 9.71501 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1769.0, - "2": 1730.0, - "3": 1728.0, - "4": 1612.0, + "1": 1725.0, + "2": 1782.0, + "3": 1740.0, + "4": 1671.0, "5": 1692.0, - "6": 1719.0, - "7": 1872.0, - "8": 1657.0, - "9": 1770.0, - "10": 1777.0, - "11": 1697.0, - "12": 1721.0, - "13": 1783.0, - "14": 1784.0, - "15": 1583.0, - "16": 1711.0, - "17": 1743.0, - "18": 1766.0, - "19": 1755.0, - "20": 1676.0, - "21": 1838.0, - "22": 1726.0, - "23": 1650.0, - "24": 1801.0, - "25": 1822.0, - "26": 1818.0, - "27": 1857.0, - "28": 1830.0, - "29": 2024.0, - "30": 1772.0, - "31": 2004.0, - "32": 1984.0, - "33": 1939.0, - "34": 2014.0, - "35": 2190.0, - "36": 2029.0, - "37": 2171.0, - "38": 2076.0, - "39": 2307.0, - "40": 2283.0, - "41": 2352.0, - "42": 2038.0, - "43": 2462.0, - "44": 2201.0, - "45": 2496.0, - "46": 2332.0, - "47": 2431.0, - "48": 2657.0, - "49": 2706.0, - "50": 2523.0, - "51": 2422.0, - "52": 2630.0, - "53": 2633.0, - "54": 2831.0, - "55": 2584.0, - "56": 2781.0, - "57": 2244.0, - "58": 3551.0, - "59": 3006.0, - "60": 2928.0, - "61": 2894.0, - "62": 3200.0, - "63": 3409.0, - "64": 3588.0, - "65": 2745.0, - "66": 3199.0, - "67": 3933.0, - "68": 3489.0, - "69": 3109.0, - "70": 3355.0, - "71": 3067.0, - "72": 2965.0, - "73": 3434.0, - "74": 3343.0, - "75": 3203.0, - "76": 3315.0, - "77": 3776.0, - "78": 3323.0, - "79": 3260.0, - "80": 3268.0, - "81": 3533.0, - "82": 2993.0, - "83": 3035.0, - "84": 2907.0, - "85": 2822.0, - "86": 2891.0, - "87": 2942.0, - "88": 3135.0, - "89": 3215.0, - "90": 3838.0, - "91": 2968.0, - "92": 2759.0, - "93": 3024.0, - "94": 2973.0, - "95": 3242.0, - "96": 3295.0, - "97": 3308.0, - "98": 3192.0, - "99": 3267.0, - "100": 3362.0 + "6": 1676.0, + "7": 1929.0, + "8": 1720.0, + "9": 1785.0, + "10": 1742.0, + "11": 1670.0, + "12": 1737.0, + "13": 1795.0, + "14": 1918.0, + "15": 1547.0, + "16": 1719.0, + "17": 1865.0, + "18": 1875.0, + "19": 1679.0, + "20": 1720.0, + "21": 1731.0, + "22": 1688.0, + "23": 1661.0, + "24": 1764.0, + "25": 1707.0, + "26": 1885.0, + "27": 1793.0, + "28": 1739.0, + "29": 2016.0, + "30": 1881.0, + "31": 2021.0, + "32": 1964.0, + "33": 1920.0, + "34": 2041.0, + "35": 2165.0, + "36": 1937.0, + "37": 2235.0, + "38": 2103.0, + "39": 2236.0, + "40": 2307.0, + "41": 2386.0, + "42": 1985.0, + "43": 2454.0, + "44": 2209.0, + "45": 2458.0, + "46": 2418.0, + "47": 2407.0, + "48": 2631.0, + "49": 2875.0, + "50": 2608.0, + "51": 2459.0, + "52": 2730.0, + "53": 2675.0, + "54": 2815.0, + "55": 2609.0, + "56": 2717.0, + "57": 2284.0, + "58": 3631.0, + "59": 2852.0, + "60": 3061.0, + "61": 2899.0, + "62": 3199.0, + "63": 3452.0, + "64": 3583.0, + "65": 2865.0, + "66": 3228.0, + "67": 4159.0, + "68": 3369.0, + "69": 3139.0, + "70": 3416.0, + "71": 3141.0, + "72": 2939.0, + "73": 3516.0, + "74": 3368.0, + "75": 3165.0, + "76": 3242.0, + "77": 3852.0, + "78": 3411.0, + "79": 3341.0, + "80": 3118.0, + "81": 3689.0, + "82": 2867.0, + "83": 3164.0, + "84": 2952.0, + "85": 2814.0, + "86": 2865.0, + "87": 2967.0, + "88": 3115.0, + "89": 3211.0, + "90": 3736.0, + "91": 2846.0, + "92": 2832.0, + "93": 3031.0, + "94": 2966.0, + "95": 3381.0, + "96": 3246.0, + "97": 3364.0, + "98": 3195.0, + "99": 3301.0, + "100": 3320.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.79237, - "3": 1.60566, - "4": 1.20579, - "5": 0.99309, - "6": 1.33557, - "7": 1.39425, - "8": 1.47423, - "9": 1.60284, - "10": 1.12246, - "11": 1.15703, - "12": 1.16144, - "13": 1.32006, - "14": 1.47244, - "15": 0.9489, - "16": 1.11446, - "17": 1.47698, - "18": 1.25144, - "19": 1.27479, - "20": 1.27309, - "21": 1.29416, - "22": 1.21734, - "23": 1.28426, - "24": 1.8034, - "25": 0.76424, - "26": 1.08554, - "27": 1.0629, - "28": 1.54008, - "29": 0.95585, - "30": 0.88659, - "31": 0.98651, - "32": 1.67568, - "33": 1.13253, - "34": 0.45273, - "35": 1.487, - "36": 1.22764, - "37": 1.10544, - "38": 0.77458, - "39": 1.48211, - "40": 1.77497, - "41": 1.38665, - "42": 1.43913, - "43": 1.09094, - "44": 1.5532, - "45": 1.32776, - "46": 0.9677, - "47": 1.28294, - "48": 1.07283, - "49": 0.91723, - "50": 1.24133, - "51": 0.39892, - "52": 1.19418, - "53": 1.33471, - "54": 1.49197, - "55": 1.32517, - "56": 1.14412, - "57": 0.87043, - "58": 0.91378, - "59": 1.19857, - "60": 1.00932, - "61": 1.10432, - "62": 1.69769, - "63": 1.16786, - "64": 1.28708, - "65": 0.7877, - "66": 1.46358, - "67": 1.3682, - "68": 1.01245, - "69": 1.45405, - "70": 0.98541, - "71": 1.12885, - "72": 1.09485, - "73": 0.96702, - "74": 0.7814, - "75": 0.92313, - "76": 1.42561, - "77": 1.31904, - "78": 1.36731, - "79": 1.95785, - "80": 1.35427, - "81": 1.11772, - "82": 1.63776, - "83": 1.3614, - "84": 1.02251, - "85": 0.80518, - "86": 1.04223, - "87": 1.94232, - "88": 1.13576, - "89": 2.26714, - "90": 1.33072, - "91": 1.53862, - "92": 1.54225, - "93": 1.26952, - "94": 1.67887, - "95": 0.9208, - "96": 1.14527, - "97": 1.08586, - "98": 0.92, - "99": 1.74351, - "100": 1.20956 + "2": 4.52535, + "3": 1.51665, + "4": 1.05653, + "5": 0.85747, + "6": 1.23751, + "7": 1.17908, + "8": 1.38858, + "9": 1.4078, + "10": 0.96403, + "11": 1.04658, + "12": 1.00099, + "13": 1.16073, + "14": 1.33531, + "15": 0.86253, + "16": 0.93498, + "17": 1.18267, + "18": 1.05638, + "19": 1.12317, + "20": 1.18207, + "21": 1.17505, + "22": 1.17146, + "23": 1.26765, + "24": 1.65361, + "25": 0.83684, + "26": 0.81628, + "27": 0.94716, + "28": 1.23309, + "29": 0.85514, + "30": 0.94315, + "31": 0.799, + "32": 1.79727, + "33": 1.14906, + "34": 0.38576, + "35": 1.34322, + "36": 1.36467, + "37": 0.99503, + "38": 0.73797, + "39": 1.15808, + "40": 1.65228, + "41": 1.13363, + "42": 1.26633, + "43": 0.82284, + "44": 1.58014, + "45": 1.19973, + "46": 0.88743, + "47": 1.27826, + "48": 1.15874, + "49": 1.03462, + "50": 1.64511, + "51": 0.38008, + "52": 1.0932, + "53": 1.36416, + "54": 1.38841, + "55": 1.41168, + "56": 1.12054, + "57": 0.86781, + "58": 0.99595, + "59": 1.20593, + "60": 1.10697, + "61": 0.98508, + "62": 1.61868, + "63": 1.33583, + "64": 1.15674, + "65": 0.94814, + "66": 1.26081, + "67": 1.35424, + "68": 1.21914, + "69": 1.49926, + "70": 1.14538, + "71": 1.34571, + "72": 1.13997, + "73": 1.03401, + "74": 0.76285, + "75": 0.88848, + "76": 1.49975, + "77": 1.51211, + "78": 1.49394, + "79": 1.93962, + "80": 1.17675, + "81": 1.06105, + "82": 1.48506, + "83": 1.30112, + "84": 0.96501, + "85": 0.92069, + "86": 1.02397, + "87": 1.89881, + "88": 1.12165, + "89": 1.05685, + "90": 1.27444, + "91": 1.11924, + "92": 1.38523, + "93": 1.16139, + "94": 1.62137, + "95": 1.06766, + "96": 1.23088, + "97": 1.16181, + "98": 0.92511, + "99": 1.57871, + "100": 1.19723 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json index b327af5c817..a9402975d7b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 637582848.0, - "2": 637582848.0, - "3": 637582848.0, - "4": 637582848.0, - "5": 637582848.0, - "6": 637582848.0, - "7": 637582848.0, - "8": 637582848.0, - "9": 637582848.0, - "10": 637582848.0, - "11": 637582848.0, - "12": 637582848.0, - "13": 637582848.0, - "14": 637582848.0, - "15": 637582848.0, - "16": 637582848.0, - "17": 637582848.0, - "18": 637582848.0, - "19": 637582848.0, - "20": 637582848.0, - "21": 637582848.0, - "22": 637582848.0, - "23": 637582848.0, - "24": 637582848.0, - "25": 637582848.0, - "26": 637582848.0, - "27": 637582848.0, - "28": 637582848.0, - "29": 637582848.0, - "30": 637582848.0, - "31": 637582848.0, - "32": 637582848.0, - "33": 637582848.0, - "34": 637582848.0, - "35": 637582848.0, - "36": 637582848.0, - "37": 637582848.0, - "38": 637582848.0, - "39": 637582848.0, - "40": 637582848.0, - "41": 637582848.0, - "42": 637582848.0, - "43": 637582848.0, - "44": 637582848.0, - "45": 637582848.0, - "46": 637582848.0, - "47": 637582848.0, - "48": 637582848.0, - "49": 637582848.0, - "50": 637582848.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 910503936.0, - "2": 1172074496.0, - "3": 1172075008.0, - "4": 1172075008.0, - "5": 1172075008.0, - "6": 1176268800.0, - "7": 1176268800.0, - "8": 1176268800.0, - "9": 1176268800.0, - "10": 1176268800.0, - "11": 1176268800.0, - "12": 1176268800.0, - "13": 1176268800.0, - "14": 1177316352.0, - "15": 1177316352.0, - "16": 1177316352.0, - "17": 1177316352.0, - "18": 1177316352.0, - "19": 1177316352.0, - "20": 1177316352.0, - "21": 1177316352.0, - "22": 1177316352.0, - "23": 1177316352.0, - "24": 1177316352.0, - "25": 1177316352.0, - "26": 1177316352.0, - "27": 1177316352.0, - "28": 1177316352.0, - "29": 1177316352.0, - "30": 1177316352.0, - "31": 1177316352.0, - "32": 1177316352.0, - "33": 1177316352.0, - "34": 1177316352.0, - "35": 1177316352.0, - "36": 1177316352.0, - "37": 1177316352.0, - "38": 1177316352.0, - "39": 1177316352.0, - "40": 1177316352.0, - "41": 1177316352.0, - "42": 1177316352.0, - "43": 1177316352.0, - "44": 1177316352.0, - "45": 1177316352.0, - "46": 1177316352.0, - "47": 1177316352.0, - "48": 1177316352.0, - "49": 1177316352.0, - "50": 1177316352.0 + "1": 971190784.0, + "2": 1231710208.0, + "3": 1231710208.0, + "4": 1231710208.0, + "5": 1231712256.0, + "6": 1231712256.0, + "7": 1231712256.0, + "8": 1231712256.0, + "9": 1231712256.0, + "10": 1231712256.0, + "11": 1231712768.0, + "12": 1231712768.0, + "13": 1232497664.0, + "14": 1232497664.0, + "15": 1232497664.0, + "16": 1232497664.0, + "17": 1232497664.0, + "18": 1232498688.0, + "19": 1232498688.0, + "20": 1232498688.0, + "21": 1232498688.0, + "22": 1232498688.0, + "23": 1232498688.0, + "24": 1232498688.0, + "25": 1232498688.0, + "26": 1232498688.0, + "27": 1232499200.0, + "28": 1232499200.0, + "29": 1232499200.0, + "30": 1232499200.0, + "31": 1232499200.0, + "32": 1232499200.0, + "33": 1232499200.0, + "34": 1232499200.0, + "35": 1232499200.0, + "36": 1232499200.0, + "37": 1232499200.0, + "38": 1232499200.0, + "39": 1232499200.0, + "40": 1232499200.0, + "41": 1232499200.0, + "42": 1232499200.0, + "43": 1232499200.0, + "44": 1232499200.0, + "45": 1232499200.0, + "46": 1232499200.0, + "47": 1232499200.0, + "48": 1232499200.0, + "49": 1232499200.0, + "50": 1232499200.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.38339, - "3": 1.82517, - "4": 1.52689, - "5": 1.15859, - "6": 1.66678, - "7": 1.55786, - "8": 2.39358, - "9": 2.27873, - "10": 1.89866, - "11": 1.87264, - "12": 1.93649, - "13": 1.96768, - "14": 2.18369, - "15": 1.77257, - "16": 1.82404, - "17": 2.50838, - "18": 1.72981, - "19": 1.87305, - "20": 1.88896, - "21": 2.21039, - "22": 1.46131, - "23": 2.11176, - "24": 1.88896, - "25": 2.26135, - "26": 2.46814, - "27": 1.99314, - "28": 2.03385, - "29": 1.68086, - "30": 1.65779, - "31": 1.65574, - "32": 2.62807, - "33": 1.57794, - "34": 1.6315, - "35": 2.39842, - "36": 2.07391, - "37": 1.76618, - "38": 1.66546, - "39": 2.21354, - "40": 1.82935, - "41": 1.82561, - "42": 2.80846, - "43": 1.69614, - "44": 1.95422, - "45": 2.31591, - "46": 1.69895, - "47": 1.76076, - "48": 2.09921, - "49": 1.63754, - "50": 1.84687 + "2": 5.89917, + "3": 2.35381, + "4": 1.65759, + "5": 1.58647, + "6": 2.1062, + "7": 1.35555, + "8": 2.08989, + "9": 2.43268, + "10": 2.267, + "11": 2.11284, + "12": 1.54632, + "13": 1.84941, + "14": 2.28705, + "15": 1.41575, + "16": 2.77867, + "17": 2.24862, + "18": 1.64676, + "19": 1.94433, + "20": 1.68397, + "21": 2.10417, + "22": 1.3851, + "23": 2.10617, + "24": 1.58709, + "25": 2.18871, + "26": 2.2332, + "27": 1.7175, + "28": 1.80487, + "29": 1.47622, + "30": 1.35141, + "31": 1.52804, + "32": 2.59212, + "33": 1.51739, + "34": 1.18734, + "35": 2.0564, + "36": 2.23036, + "37": 1.73408, + "38": 1.47106, + "39": 2.12942, + "40": 1.59761, + "41": 1.48883, + "42": 2.09592, + "43": 1.4522, + "44": 1.78115, + "45": 2.10328, + "46": 1.72849, + "47": 1.95566, + "48": 1.87309, + "49": 1.46805, + "50": 1.75862 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json index 1304bf33e11..0f2cdd8416a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 640073216.0, - "2": 640073216.0, - "3": 640073216.0, - "4": 640073216.0, - "5": 640073216.0, - "6": 640073216.0, - "7": 640073216.0, - "8": 640073216.0, - "9": 640073216.0, - "10": 640073216.0, - "11": 640073216.0, - "12": 640073216.0, - "13": 640073216.0, - "14": 640073216.0, - "15": 640073216.0, - "16": 640073216.0, - "17": 640073216.0, - "18": 640073216.0, - "19": 640073216.0, - "20": 640073216.0, - "21": 640073216.0, - "22": 640073216.0, - "23": 640073216.0, - "24": 640073216.0, - "25": 640073216.0, - "26": 640073216.0, - "27": 640073216.0, - "28": 640073216.0, - "29": 640073216.0, - "30": 640073216.0, - "31": 640073216.0, - "32": 640073216.0, - "33": 640073216.0, - "34": 640073216.0, - "35": 640073216.0, - "36": 640073216.0, - "37": 640073216.0, - "38": 640073216.0, - "39": 640073216.0, - "40": 640073216.0, - "41": 640073216.0, - "42": 640073216.0, - "43": 640073216.0, - "44": 640073216.0, - "45": 640073216.0, - "46": 640073216.0, - "47": 640073216.0, - "48": 640073216.0, - "49": 640073216.0, - "50": 640073216.0 + "1": 688307712.0, + "2": 688307712.0, + "3": 688307712.0, + "4": 688307712.0, + "5": 688307712.0, + "6": 688307712.0, + "7": 688307712.0, + "8": 688307712.0, + "9": 688307712.0, + "10": 688307712.0, + "11": 688307712.0, + "12": 688307712.0, + "13": 688307712.0, + "14": 688307712.0, + "15": 688307712.0, + "16": 688307712.0, + "17": 688307712.0, + "18": 688307712.0, + "19": 688307712.0, + "20": 688307712.0, + "21": 688307712.0, + "22": 688307712.0, + "23": 688307712.0, + "24": 688307712.0, + "25": 688307712.0, + "26": 688307712.0, + "27": 688307712.0, + "28": 688307712.0, + "29": 688307712.0, + "30": 688307712.0, + "31": 688307712.0, + "32": 688307712.0, + "33": 688307712.0, + "34": 688307712.0, + "35": 688307712.0, + "36": 688307712.0, + "37": 688307712.0, + "38": 688307712.0, + "39": 688307712.0, + "40": 688307712.0, + "41": 688307712.0, + "42": 688307712.0, + "43": 688307712.0, + "44": 688307712.0, + "45": 688307712.0, + "46": 688307712.0, + "47": 688307712.0, + "48": 688307712.0, + "49": 688307712.0, + "50": 688307712.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 909846528.0, - "2": 1170895360.0, - "3": 1183476736.0, - "4": 1183478272.0, - "5": 1183478272.0, - "6": 1183739904.0, - "7": 1183739904.0, - "8": 1183739904.0, - "9": 1183739904.0, - "10": 1183739904.0, - "11": 1183739904.0, - "12": 1183739904.0, - "13": 1183739904.0, - "14": 1183739904.0, - "15": 1184393216.0, - "16": 1184393216.0, - "17": 1184393216.0, - "18": 1184393216.0, - "19": 1184393216.0, - "20": 1184393216.0, - "21": 1184393216.0, - "22": 1184393216.0, - "23": 1184393216.0, - "24": 1184393216.0, - "25": 1184393216.0, - "26": 1184393216.0, - "27": 1184393216.0, - "28": 1184393216.0, - "29": 1184393216.0, - "30": 1184393216.0, - "31": 1184393216.0, - "32": 1184393216.0, - "33": 1184393216.0, - "34": 1184393216.0, - "35": 1184393216.0, - "36": 1184393216.0, - "37": 1184393216.0, - "38": 1184393216.0, - "39": 1184393216.0, - "40": 1184393216.0, - "41": 1184393216.0, - "42": 1184393216.0, - "43": 1184393216.0, - "44": 1184393216.0, - "45": 1184393216.0, - "46": 1184393216.0, - "47": 1184393216.0, - "48": 1184393216.0, - "49": 1184393216.0, - "50": 1184393216.0 + "1": 968044032.0, + "2": 1224372736.0, + "3": 1228567040.0, + "4": 1228567040.0, + "5": 1228567040.0, + "6": 1228567040.0, + "7": 1228567040.0, + "8": 1228567040.0, + "9": 1228567040.0, + "10": 1228567040.0, + "11": 1228567040.0, + "12": 1231710208.0, + "13": 1231710208.0, + "14": 1231710208.0, + "15": 1231710208.0, + "16": 1231710208.0, + "17": 1231710208.0, + "18": 1231710208.0, + "19": 1231710208.0, + "20": 1231710208.0, + "21": 1231710208.0, + "22": 1231712256.0, + "23": 1231712256.0, + "24": 1231712256.0, + "25": 1231712256.0, + "26": 1231712256.0, + "27": 1231712256.0, + "28": 1231712256.0, + "29": 1231712256.0, + "30": 1231712256.0, + "31": 1231712256.0, + "32": 1231712256.0, + "33": 1231712256.0, + "34": 1231712768.0, + "35": 1231712768.0, + "36": 1231712768.0, + "37": 1231712768.0, + "38": 1231712768.0, + "39": 1231712768.0, + "40": 1231712768.0, + "41": 1231712768.0, + "42": 1231712768.0, + "43": 1231712768.0, + "44": 1231712768.0, + "45": 1231712768.0, + "46": 1231712768.0, + "47": 1231712768.0, + "48": 1231712768.0, + "49": 1231712768.0, + "50": 1231712768.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.12844, - "3": 2.21371, - "4": 1.92237, - "5": 1.6034, - "6": 2.00558, - "7": 1.38797, - "8": 2.38842, - "9": 2.43381, - "10": 1.84294, - "11": 2.16314, - "12": 2.01399, - "13": 1.99068, - "14": 2.28159, - "15": 1.72273, - "16": 2.05062, - "17": 2.2891, - "18": 1.71064, - "19": 1.89348, - "20": 1.98018, - "21": 1.84498, - "22": 1.66919, - "23": 2.07264, - "24": 1.88944, - "25": 2.23371, - "26": 2.25357, - "27": 1.75533, - "28": 1.69791, - "29": 1.48707, - "30": 1.64585, - "31": 1.62523, - "32": 2.5903, - "33": 1.64794, - "34": 1.52985, - "35": 2.35537, - "36": 2.25158, - "37": 1.77292, - "38": 1.55473, - "39": 2.04974, - "40": 1.99804, - "41": 1.99066, - "42": 2.49355, - "43": 1.53976, - "44": 2.21525, - "45": 2.01832, - "46": 1.83985, - "47": 1.62482, - "48": 1.97152, - "49": 1.6326, - "50": 1.95741 + "2": 5.93351, + "3": 1.76566, + "4": 1.58706, + "5": 1.46021, + "6": 1.84323, + "7": 1.52377, + "8": 2.03454, + "9": 2.27232, + "10": 1.86264, + "11": 2.03937, + "12": 1.87444, + "13": 2.04501, + "14": 1.92326, + "15": 1.66282, + "16": 1.60579, + "17": 1.9194, + "18": 1.50956, + "19": 1.86947, + "20": 1.75827, + "21": 2.13354, + "22": 1.57839, + "23": 1.88712, + "24": 1.839, + "25": 2.10635, + "26": 1.87152, + "27": 1.7463, + "28": 1.64562, + "29": 1.50717, + "30": 1.72891, + "31": 1.7433, + "32": 2.43911, + "33": 1.64575, + "34": 1.54187, + "35": 2.13342, + "36": 1.66954, + "37": 1.59523, + "38": 1.92727, + "39": 1.81438, + "40": 2.02765, + "41": 1.79363, + "42": 2.06476, + "43": 1.71864, + "44": 1.81633, + "45": 1.92118, + "46": 1.15264, + "47": 1.87413, + "48": 1.8917, + "49": 1.76553, + "50": 1.64804 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json index 9b53019bf46..9e76500d464 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 640073216.0, - "2": 640073216.0, - "3": 640073216.0, - "4": 640073216.0, - "5": 640073216.0, - "6": 640073216.0, - "7": 640073216.0, - "8": 640073216.0, - "9": 640073216.0, - "10": 640073216.0, - "11": 640073216.0, - "12": 640073216.0, - "13": 640073216.0, - "14": 640073216.0, - "15": 640073216.0, - "16": 640073216.0, - "17": 640073216.0, - "18": 640073216.0, - "19": 640073216.0, - "20": 640073216.0, - "21": 640073216.0, - "22": 640073216.0, - "23": 640073216.0, - "24": 640073216.0, - "25": 640073216.0, - "26": 640073216.0, - "27": 640073216.0, - "28": 640073216.0, - "29": 640073216.0, - "30": 640073216.0, - "31": 640073216.0, - "32": 640073216.0, - "33": 640073216.0, - "34": 640073216.0, - "35": 640073216.0, - "36": 640073216.0, - "37": 640073216.0, - "38": 640073216.0, - "39": 640073216.0, - "40": 640073216.0, - "41": 640073216.0, - "42": 640073216.0, - "43": 640073216.0, - "44": 640073216.0, - "45": 640073216.0, - "46": 640073216.0, - "47": 640073216.0, - "48": 640073216.0, - "49": 640073216.0, - "50": 640073216.0 + "1": 688307712.0, + "2": 688307712.0, + "3": 688307712.0, + "4": 688307712.0, + "5": 688307712.0, + "6": 688307712.0, + "7": 688307712.0, + "8": 688307712.0, + "9": 688307712.0, + "10": 688307712.0, + "11": 688307712.0, + "12": 688307712.0, + "13": 688307712.0, + "14": 688307712.0, + "15": 688307712.0, + "16": 688307712.0, + "17": 688307712.0, + "18": 688307712.0, + "19": 688307712.0, + "20": 688307712.0, + "21": 688307712.0, + "22": 688307712.0, + "23": 688307712.0, + "24": 688307712.0, + "25": 688307712.0, + "26": 688307712.0, + "27": 688307712.0, + "28": 688307712.0, + "29": 688307712.0, + "30": 688307712.0, + "31": 688307712.0, + "32": 688307712.0, + "33": 688307712.0, + "34": 688307712.0, + "35": 688307712.0, + "36": 688307712.0, + "37": 688307712.0, + "38": 688307712.0, + "39": 688307712.0, + "40": 688307712.0, + "41": 688307712.0, + "42": 688307712.0, + "43": 688307712.0, + "44": 688307712.0, + "45": 688307712.0, + "46": 688307712.0, + "47": 688307712.0, + "48": 688307712.0, + "49": 688307712.0, + "50": 688307712.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 910371328.0, - "2": 1183478272.0, - "3": 1184393216.0, - "4": 1184393216.0, - "5": 1184393216.0, - "6": 1184393216.0, - "7": 1184393216.0, - "8": 1184393216.0, - "9": 1184393216.0, - "10": 1184394752.0, - "11": 1184394752.0, - "12": 1184394752.0, - "13": 1184394752.0, - "14": 1184394752.0, - "15": 1184394752.0, - "16": 1184394752.0, - "17": 1184394752.0, - "18": 1184394752.0, - "19": 1184394752.0, - "20": 1184394752.0, - "21": 1184394752.0, - "22": 1184394752.0, - "23": 1184394752.0, - "24": 1184394752.0, - "25": 1184394752.0, - "26": 1184394752.0, - "27": 1184394752.0, - "28": 1184394752.0, - "29": 1184394752.0, - "30": 1184394752.0, - "31": 1184394752.0, - "32": 1184394752.0, - "33": 1184394752.0, - "34": 1184394752.0, - "35": 1184394752.0, - "36": 1184394752.0, - "37": 1184394752.0, - "38": 1184394752.0, - "39": 1184394752.0, - "40": 1184394752.0, - "41": 1184394752.0, - "42": 1184394752.0, - "43": 1184394752.0, - "44": 1184394752.0, - "45": 1184394752.0, - "46": 1184394752.0, - "47": 1184394752.0, - "48": 1184394752.0, - "49": 1184394752.0, - "50": 1184394752.0 + "1": 961752576.0, + "2": 1226467328.0, + "3": 1226467328.0, + "4": 1228564992.0, + "5": 1228564992.0, + "6": 1231711232.0, + "7": 1231711232.0, + "8": 1231711232.0, + "9": 1231711232.0, + "10": 1231711232.0, + "11": 1231711232.0, + "12": 1231712256.0, + "13": 1231712256.0, + "14": 1231712256.0, + "15": 1231712256.0, + "16": 1231712256.0, + "17": 1231712256.0, + "18": 1231712768.0, + "19": 1231712768.0, + "20": 1231712768.0, + "21": 1231712768.0, + "22": 1231712768.0, + "23": 1231712768.0, + "24": 1231712768.0, + "25": 1231712768.0, + "26": 1231712768.0, + "27": 1231712768.0, + "28": 1231712768.0, + "29": 1231712768.0, + "30": 1231712768.0, + "31": 1231712768.0, + "32": 1231712768.0, + "33": 1231712768.0, + "34": 1231712768.0, + "35": 1231712768.0, + "36": 1231712768.0, + "37": 1231712768.0, + "38": 1231712768.0, + "39": 1231712768.0, + "40": 1231712768.0, + "41": 1231712768.0, + "42": 1231712768.0, + "43": 1231712768.0, + "44": 1232105472.0, + "45": 1232105472.0, + "46": 1232105472.0, + "47": 1232105472.0, + "48": 1232105472.0, + "49": 1232105472.0, + "50": 1232105472.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.04512, - "3": 2.53622, - "4": 2.14187, - "5": 1.70192, - "6": 2.20905, - "7": 1.87249, - "8": 2.24231, - "9": 2.04395, - "10": 1.95361, - "11": 2.16036, - "12": 1.74772, - "13": 1.9621, - "14": 2.19091, - "15": 1.86195, - "16": 1.75546, - "17": 2.37569, - "18": 1.86685, - "19": 1.80379, - "20": 1.6236, - "21": 2.02964, - "22": 1.8257, - "23": 2.23803, - "24": 1.59366, - "25": 2.45307, - "26": 2.38901, - "27": 1.83894, - "28": 1.92709, - "29": 1.52188, - "30": 1.73958, - "31": 1.65113, - "32": 3.11449, - "33": 1.54333, - "34": 1.71238, - "35": 2.41273, - "36": 2.26339, - "37": 1.89629, - "38": 1.57503, - "39": 2.4608, - "40": 2.11154, - "41": 2.11892, - "42": 2.642, - "43": 1.72126, - "44": 1.96065, - "45": 2.43263, - "46": 1.83922, - "47": 2.06797, - "48": 1.96396, - "49": 1.58463, - "50": 2.07216 + "2": 6.74227, + "3": 1.93297, + "4": 1.81904, + "5": 1.64272, + "6": 1.89352, + "7": 1.72245, + "8": 2.11346, + "9": 2.31075, + "10": 1.91748, + "11": 1.92367, + "12": 2.05351, + "13": 1.65379, + "14": 2.20996, + "15": 1.68387, + "16": 2.08383, + "17": 2.59878, + "18": 2.01555, + "19": 1.87457, + "20": 1.44736, + "21": 2.41589, + "22": 1.62698, + "23": 2.1516, + "24": 1.60382, + "25": 2.03031, + "26": 2.24763, + "27": 1.83551, + "28": 1.89482, + "29": 1.64569, + "30": 1.68165, + "31": 1.85995, + "32": 2.8569, + "33": 1.8979, + "34": 1.87734, + "35": 2.42821, + "36": 2.65766, + "37": 2.38268, + "38": 1.7225, + "39": 2.28065, + "40": 2.04408, + "41": 2.20927, + "42": 2.44102, + "43": 2.33851, + "44": 2.04334, + "45": 2.64463, + "46": 1.92252, + "47": 2.02345, + "48": 2.38311, + "49": 1.70282, + "50": 2.18837 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json index b3da7a4d92b..32585f67755 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.95727, "2": 10.94891, - "3": 10.95181, + "3": 10.95177, "4": 10.93701, - "5": 10.94341, + "5": 10.94343, "6": 10.9381, - "7": 10.94906, - "8": 10.92913, - "9": 10.94291, + "7": 10.94905, + "8": 10.92918, + "9": 10.9429, "10": 10.93268, "11": 10.9326, - "12": 10.92193, - "13": 10.91841, - "14": 10.91226, - "15": 10.891, - "16": 10.87484, - "17": 10.88661, - "18": 10.86792, - "19": 10.87319, - "20": 10.77508, - "21": 10.76177, - "22": 10.75197, - "23": 10.73997, - "24": 10.70111, - "25": 10.71372, - "26": 10.68547, - "27": 10.64309, - "28": 10.56423, - "29": 10.53588, - "30": 10.52933, - "31": 10.51593, + "12": 10.92192, + "13": 10.91842, + "14": 10.91222, + "15": 10.89101, + "16": 10.87481, + "17": 10.88664, + "18": 10.86793, + "19": 10.87317, + "20": 10.77507, + "21": 10.7618, + "22": 10.75192, + "23": 10.73995, + "24": 10.70105, + "25": 10.71371, + "26": 10.68548, + "27": 10.64307, + "28": 10.56424, + "29": 10.53591, + "30": 10.52935, + "31": 10.51595, "32": 10.49935, - "33": 10.46546, - "34": 10.42362, - "35": 10.42255, - "36": 10.41397, + "33": 10.46547, + "34": 10.42361, + "35": 10.4226, + "36": 10.41393, "37": 10.36771, - "38": 10.37996, + "38": 10.37994, "39": 10.34012, - "40": 10.32558, - "41": 10.30513, + "40": 10.32557, + "41": 10.30512, "42": 10.29477, "43": 10.25699, - "44": 10.23372, - "45": 10.2453, - "46": 10.21282, - "47": 10.19585, - "48": 10.15323, + "44": 10.23375, + "45": 10.24532, + "46": 10.21281, + "47": 10.19586, + "48": 10.15327, "49": 10.14471, - "50": 10.15149 + "50": 10.15153 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 652.0, - "2": 628.0, - "3": 651.0, - "4": 603.0, - "5": 648.0, - "6": 672.0, - "7": 646.0, - "8": 637.0, - "9": 647.0, - "10": 666.0, - "11": 630.0, - "12": 627.0, - "13": 648.0, - "14": 598.0, - "15": 603.0, - "16": 597.0, - "17": 647.0, - "18": 570.0, - "19": 581.0, - "20": 612.0, - "21": 628.0, - "22": 681.0, - "23": 689.0, - "24": 651.0, - "25": 653.0, - "26": 660.0, - "27": 668.0, - "28": 670.0, - "29": 787.0, - "30": 654.0, - "31": 739.0, - "32": 721.0, - "33": 751.0, - "34": 729.0, - "35": 794.0, - "36": 819.0, - "37": 818.0, - "38": 785.0, - "39": 803.0, - "40": 874.0, - "41": 860.0, - "42": 726.0, - "43": 861.0, - "44": 774.0, - "45": 953.0, - "46": 942.0, - "47": 867.0, - "48": 920.0, - "49": 986.0, - "50": 993.0 + "1": 638.0, + "2": 640.0, + "3": 649.0, + "4": 640.0, + "5": 606.0, + "6": 649.0, + "7": 647.0, + "8": 666.0, + "9": 641.0, + "10": 684.0, + "11": 605.0, + "12": 611.0, + "13": 616.0, + "14": 627.0, + "15": 609.0, + "16": 626.0, + "17": 643.0, + "18": 677.0, + "19": 621.0, + "20": 607.0, + "21": 626.0, + "22": 650.0, + "23": 619.0, + "24": 630.0, + "25": 636.0, + "26": 638.0, + "27": 688.0, + "28": 667.0, + "29": 713.0, + "30": 719.0, + "31": 774.0, + "32": 769.0, + "33": 796.0, + "34": 825.0, + "35": 797.0, + "36": 769.0, + "37": 797.0, + "38": 816.0, + "39": 781.0, + "40": 829.0, + "41": 855.0, + "42": 763.0, + "43": 913.0, + "44": 910.0, + "45": 941.0, + "46": 889.0, + "47": 932.0, + "48": 973.0, + "49": 1085.0, + "50": 1004.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 638631424.0, - "2": 638631424.0, - "3": 638631424.0, - "4": 638631424.0, - "5": 638631424.0, - "6": 638631424.0, - "7": 638631424.0, - "8": 638631424.0, - "9": 638631424.0, - "10": 638631424.0, - "11": 638631424.0, - "12": 638631424.0, - "13": 638631424.0, - "14": 638631424.0, - "15": 638631424.0, - "16": 638631424.0, - "17": 638631424.0, - "18": 638631424.0, - "19": 638631424.0, - "20": 638631424.0, - "21": 638631424.0, - "22": 638631424.0, - "23": 638631424.0, - "24": 638631424.0, - "25": 638631424.0, - "26": 638631424.0, - "27": 638631424.0, - "28": 638631424.0, - "29": 638631424.0, - "30": 638631424.0, - "31": 638631424.0, - "32": 638631424.0, - "33": 638631424.0, - "34": 638631424.0, - "35": 638631424.0, - "36": 638631424.0, - "37": 638631424.0, - "38": 638631424.0, - "39": 638631424.0, - "40": 638631424.0, - "41": 638631424.0, - "42": 638631424.0, - "43": 638631424.0, - "44": 638631424.0, - "45": 638631424.0, - "46": 638631424.0, - "47": 638631424.0, - "48": 638631424.0, - "49": 638631424.0, - "50": 638631424.0 + "1": 689356288.0, + "2": 689356288.0, + "3": 689356288.0, + "4": 689356288.0, + "5": 689356288.0, + "6": 689356288.0, + "7": 689356288.0, + "8": 689356288.0, + "9": 689356288.0, + "10": 689356288.0, + "11": 689356288.0, + "12": 689356288.0, + "13": 689356288.0, + "14": 689356288.0, + "15": 689356288.0, + "16": 689356288.0, + "17": 689356288.0, + "18": 689356288.0, + "19": 689356288.0, + "20": 689356288.0, + "21": 689356288.0, + "22": 689356288.0, + "23": 689356288.0, + "24": 689356288.0, + "25": 689356288.0, + "26": 689356288.0, + "27": 689356288.0, + "28": 689356288.0, + "29": 689356288.0, + "30": 689356288.0, + "31": 689356288.0, + "32": 689356288.0, + "33": 689356288.0, + "34": 689356288.0, + "35": 689356288.0, + "36": 689356288.0, + "37": 689356288.0, + "38": 689356288.0, + "39": 689356288.0, + "40": 689356288.0, + "41": 689356288.0, + "42": 689356288.0, + "43": 689356288.0, + "44": 689356288.0, + "45": 689356288.0, + "46": 689356288.0, + "47": 689356288.0, + "48": 689356288.0, + "49": 689356288.0, + "50": 689356288.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 911682560.0, - "2": 1170501120.0, - "3": 1177317376.0, - "4": 1177317376.0, - "5": 1177317376.0, - "6": 1177317376.0, - "7": 1177317376.0, - "8": 1177839104.0, - "9": 1177839104.0, - "10": 1177839104.0, - "11": 1177839104.0, - "12": 1177839104.0, - "13": 1177839104.0, - "14": 1177839104.0, - "15": 1177839104.0, - "16": 1177839104.0, - "17": 1177839104.0, - "18": 1177839104.0, - "19": 1177839104.0, - "20": 1177839104.0, - "21": 1177839104.0, - "22": 1177839104.0, - "23": 1177839104.0, - "24": 1177839104.0, - "25": 1177839104.0, - "26": 1177839104.0, - "27": 1177839104.0, - "28": 1177839104.0, - "29": 1177839104.0, - "30": 1177839104.0, - "31": 1177839104.0, - "32": 1177839104.0, - "33": 1177839104.0, - "34": 1177839104.0, - "35": 1177839104.0, - "36": 1177839104.0, - "37": 1177839104.0, - "38": 1180463616.0, - "39": 1180463616.0, - "40": 1180463616.0, - "41": 1180463616.0, - "42": 1180463616.0, - "43": 1180463616.0, - "44": 1180463616.0, - "45": 1180463616.0, - "46": 1180463616.0, - "47": 1180463616.0, - "48": 1180463616.0, - "49": 1180463616.0, - "50": 1180463616.0 + "1": 971190784.0, + "2": 1224372224.0, + "3": 1224372224.0, + "4": 1224372224.0, + "5": 1224372224.0, + "6": 1231711232.0, + "7": 1231711232.0, + "8": 1231711232.0, + "9": 1231711232.0, + "10": 1231711232.0, + "11": 1231711232.0, + "12": 1232496640.0, + "13": 1232496640.0, + "14": 1232496640.0, + "15": 1232496640.0, + "16": 1232496640.0, + "17": 1232496640.0, + "18": 1232496640.0, + "19": 1232496640.0, + "20": 1232499200.0, + "21": 1232499200.0, + "22": 1232499200.0, + "23": 1232499200.0, + "24": 1232499200.0, + "25": 1232499200.0, + "26": 1232499200.0, + "27": 1232499200.0, + "28": 1232499200.0, + "29": 1232499200.0, + "30": 1232499200.0, + "31": 1232499200.0, + "32": 1232499200.0, + "33": 1232499200.0, + "34": 1232499200.0, + "35": 1232499200.0, + "36": 1232499200.0, + "37": 1232499200.0, + "38": 1232499200.0, + "39": 1232499200.0, + "40": 1232499200.0, + "41": 1232499200.0, + "42": 1232499200.0, + "43": 1232499200.0, + "44": 1232499200.0, + "45": 1232499200.0, + "46": 1232499200.0, + "47": 1232499200.0, + "48": 1232499200.0, + "49": 1232499200.0, + "50": 1232499200.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.20715, - "3": 1.92588, - "4": 1.7209, - "5": 1.26617, - "6": 1.91225, - "7": 1.40677, - "8": 1.68753, - "9": 2.14463, - "10": 1.9968, - "11": 1.88521, - "12": 1.94345, - "13": 1.62284, - "14": 1.87583, - "15": 1.78938, - "16": 1.49862, - "17": 1.79407, - "18": 1.6566, - "19": 1.73373, - "20": 1.65499, - "21": 1.65561, - "22": 1.48916, - "23": 1.58948, - "24": 1.43527, - "25": 1.64856, - "26": 1.68885, - "27": 1.42867, - "28": 1.39778, - "29": 1.51816, - "30": 1.39869, - "31": 1.19871, - "32": 2.35599, - "33": 1.4144, - "34": 1.20483, - "35": 1.8763, - "36": 1.71344, - "37": 1.37824, - "38": 1.53973, - "39": 2.04209, - "40": 1.79838, - "41": 2.16606, - "42": 2.00263, - "43": 1.67464, - "44": 1.61536, - "45": 2.20218, - "46": 1.37692, - "47": 1.49279, - "48": 1.69211, - "49": 1.39893, - "50": 1.8565 + "2": 6.24341, + "3": 2.05909, + "4": 1.65052, + "5": 1.20871, + "6": 1.97414, + "7": 1.21659, + "8": 1.89567, + "9": 2.09571, + "10": 1.86118, + "11": 1.50307, + "12": 1.32404, + "13": 1.56012, + "14": 1.62625, + "15": 1.24018, + "16": 1.5879, + "17": 2.11033, + "18": 1.47326, + "19": 1.59956, + "20": 1.337, + "21": 1.80514, + "22": 1.32273, + "23": 1.71041, + "24": 1.11581, + "25": 1.62306, + "26": 2.033, + "27": 1.61909, + "28": 1.66659, + "29": 1.24585, + "30": 1.53955, + "31": 1.17978, + "32": 2.0212, + "33": 1.11818, + "34": 1.20347, + "35": 1.66884, + "36": 1.8531, + "37": 1.38261, + "38": 1.21104, + "39": 1.80889, + "40": 1.73775, + "41": 1.83693, + "42": 2.11407, + "43": 1.64143, + "44": 1.39973, + "45": 1.92661, + "46": 1.33891, + "47": 1.4435, + "48": 1.61259, + "49": 1.55248, + "50": 1.91331 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json index 6d3a37e6e1a..2775ea7af8d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92858, - "2": 10.92691, - "3": 10.9228, - "4": 10.92385, - "5": 10.92018, + "1": 10.92859, + "2": 10.9269, + "3": 10.92279, + "4": 10.92378, + "5": 10.92022, "6": 10.92481, - "7": 10.91599, - "8": 10.92353, - "9": 10.91957, - "10": 10.92508, - "11": 10.92005, - "12": 10.92356, - "13": 10.91465, - "14": 10.91785, - "15": 10.91656, - "16": 10.92311, - "17": 10.91335, - "18": 10.91241, - "19": 10.90858, - "20": 10.90755, - "21": 10.89514, - "22": 10.89723, - "23": 10.89723, - "24": 10.89583, - "25": 10.90396, - "26": 10.89488, - "27": 10.88792, - "28": 10.86575, - "29": 10.85896, - "30": 10.85707, - "31": 10.84766, - "32": 10.86079, - "33": 10.84972, - "34": 10.83674, - "35": 10.84444, - "36": 10.83032, - "37": 10.82693, - "38": 10.80413, + "7": 10.91589, + "8": 10.92356, + "9": 10.91952, + "10": 10.92509, + "11": 10.92003, + "12": 10.92357, + "13": 10.9147, + "14": 10.91789, + "15": 10.91662, + "16": 10.92306, + "17": 10.91332, + "18": 10.9124, + "19": 10.90854, + "20": 10.9076, + "21": 10.89511, + "22": 10.89724, + "23": 10.89727, + "24": 10.89578, + "25": 10.90401, + "26": 10.89489, + "27": 10.88789, + "28": 10.86573, + "29": 10.85901, + "30": 10.85709, + "31": 10.8477, + "32": 10.86082, + "33": 10.84971, + "34": 10.83681, + "35": 10.84447, + "36": 10.83037, + "37": 10.82695, + "38": 10.80411, "39": 10.77934, - "40": 10.77162, - "41": 10.75997, - "42": 10.75927, + "40": 10.77166, + "41": 10.76003, + "42": 10.75933, "43": 10.75404, - "44": 10.74361, - "45": 10.73916, - "46": 10.71446, - "47": 10.71009, - "48": 10.68542, - "49": 10.69158, - "50": 10.6792 + "44": 10.74359, + "45": 10.73922, + "46": 10.71445, + "47": 10.71014, + "48": 10.68548, + "49": 10.6915, + "50": 10.67923 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 23025846.0, - "2": 22914698.0, - "3": 22773682.0, - "4": 22842060.0, - "5": 22847304.0, - "6": 22810108.0, - "7": 22935284.0, - "8": 22675112.0, - "9": 22822952.0, - "10": 22552496.0, - "11": 22818072.0, - "12": 22705452.0, - "13": 23386520.0, - "14": 23055528.0, - "15": 22782176.0, - "16": 22886556.0, - "17": 22992588.0, + "1": 23025860.0, + "2": 22914720.0, + "3": 22773588.0, + "4": 22842072.0, + "5": 22847252.0, + "6": 22810068.0, + "7": 22935268.0, + "8": 22675124.0, + "9": 22822992.0, + "10": 22552360.0, + "11": 22817968.0, + "12": 22705544.0, + "13": 23386596.0, + "14": 23055506.0, + "15": 22782216.0, + "16": 22886524.0, + "17": 22992648.0, "18": 23059260.0, - "19": 23148592.0, - "20": 22786778.0, - "21": 22982468.0, - "22": 23009396.0, - "23": 22693052.0, - "24": 22926420.0, - "25": 22699896.0, - "26": 23068178.0, - "27": 22865288.0, - "28": 23063142.0, - "29": 23043304.0, - "30": 23007890.0, - "31": 22972192.0, - "32": 22730568.0, - "33": 22804668.0, - "34": 23140012.0, - "35": 22814154.0, - "36": 22759064.0, - "37": 23167088.0, - "38": 23029028.0, - "39": 23048668.0, - "40": 22813504.0, - "41": 23128932.0, - "42": 22752812.0, - "43": 23050544.0, - "44": 22768146.0, - "45": 22910452.0, - "46": 22797168.0, - "47": 22912692.0, - "48": 22897264.0, - "49": 22951794.0, - "50": 22707566.0 + "19": 23148520.0, + "20": 22786856.0, + "21": 22982420.0, + "22": 23009412.0, + "23": 22693004.0, + "24": 22926362.0, + "25": 22699796.0, + "26": 23068216.0, + "27": 22865320.0, + "28": 23063088.0, + "29": 23043308.0, + "30": 23007930.0, + "31": 22972168.0, + "32": 22730592.0, + "33": 22804614.0, + "34": 23140082.0, + "35": 22814240.0, + "36": 22758996.0, + "37": 23167084.0, + "38": 23029010.0, + "39": 23048660.0, + "40": 22813540.0, + "41": 23128888.0, + "42": 22752764.0, + "43": 23050580.0, + "44": 22768264.0, + "45": 22910384.0, + "46": 22797176.0, + "47": 22912748.0, + "48": 22897280.0, + "49": 22951760.0, + "50": 22707564.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.04488, - "3": 0.14123, - "4": 0.13989, - "5": 0.13964, - "6": 0.14307, - "7": 0.14186, - "8": 0.13867, - "9": 0.13845, - "10": 0.13944, - "11": 0.13857, - "12": 0.14012, - "13": 0.14088, - "14": 0.13898, - "15": 0.13896, - "16": 0.13994, - "17": 0.13894, - "18": 0.13867, - "19": 0.13866, - "20": 0.14137, - "21": 0.13893, - "22": 0.13714, - "23": 0.14007, - "24": 0.13871, - "25": 0.14434, - "26": 0.17261, - "27": 0.16786, - "28": 0.14429, - "29": 0.14179, - "30": 0.14227, - "31": 0.14275, - "32": 0.14195, - "33": 0.1392, - "34": 0.13898, - "35": 0.13863, - "36": 0.14135, - "37": 0.13876, - "38": 0.13785, - "39": 0.1478, - "40": 0.14002, - "41": 0.13994, - "42": 0.14007, - "43": 0.13829, - "44": 0.13827, - "45": 0.13753, - "46": 0.13849, - "47": 0.14184, - "48": 0.13854, - "49": 0.13966, - "50": 0.13944 + "2": 5.60012, + "3": 0.14581, + "4": 0.14637, + "5": 0.1445, + "6": 0.14398, + "7": 0.14375, + "8": 0.14221, + "9": 0.14321, + "10": 0.14262, + "11": 0.14518, + "12": 0.14779, + "13": 0.14625, + "14": 0.14424, + "15": 0.14464, + "16": 0.14695, + "17": 0.14823, + "18": 0.14414, + "19": 0.14268, + "20": 0.14673, + "21": 0.14551, + "22": 0.14449, + "23": 0.14522, + "24": 0.14308, + "25": 0.1434, + "26": 0.18906, + "27": 0.17532, + "28": 0.1547, + "29": 0.14735, + "30": 0.14489, + "31": 0.14638, + "32": 0.14246, + "33": 0.14281, + "34": 0.14231, + "35": 0.14121, + "36": 0.14136, + "37": 0.14322, + "38": 0.14477, + "39": 0.14626, + "40": 0.14745, + "41": 0.14314, + "42": 0.14198, + "43": 0.14623, + "44": 0.14584, + "45": 0.14184, + "46": 0.14869, + "47": 0.14246, + "48": 0.1416, + "49": 0.14199, + "50": 0.14341 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json index 9fd43415bf0..da9d81ca269 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.95497, - "2": 10.95336, - "3": 10.95084, - "4": 10.95138, - "5": 10.95095, - "6": 10.95418, - "7": 10.95215, - "8": 10.95329, - "9": 10.9475, + "1": 10.95495, + "2": 10.95329, + "3": 10.95082, + "4": 10.95133, + "5": 10.9509, + "6": 10.95414, + "7": 10.95218, + "8": 10.95334, + "9": 10.94752, "10": 10.94585, - "11": 10.94813, - "12": 10.96094, - "13": 10.9515, - "14": 10.95222, - "15": 10.94823, - "16": 10.94694, - "17": 10.95226, - "18": 10.94757, - "19": 10.95164, - "20": 10.93972, - "21": 10.93985, - "22": 10.93777, + "11": 10.94815, + "12": 10.96099, + "13": 10.95154, + "14": 10.95219, + "15": 10.94822, + "16": 10.94693, + "17": 10.95232, + "18": 10.94756, + "19": 10.95156, + "20": 10.93971, + "21": 10.93986, + "22": 10.93779, "23": 10.92832, - "24": 10.92783, - "25": 10.93223, - "26": 10.92799, - "27": 10.9229, - "28": 10.89388, - "29": 10.88953, - "30": 10.8825, - "31": 10.88534, - "32": 10.88053, - "33": 10.87143, - "34": 10.86879, - "35": 10.86909, + "24": 10.92785, + "25": 10.93225, + "26": 10.92789, + "27": 10.92298, + "28": 10.89386, + "29": 10.88954, + "30": 10.88251, + "31": 10.88536, + "32": 10.88054, + "33": 10.87139, + "34": 10.86882, + "35": 10.86911, "36": 10.86039, - "37": 10.85941, - "38": 10.83806, + "37": 10.85944, + "38": 10.83809, "39": 10.80305, "40": 10.80807, "41": 10.79481, - "42": 10.77754, - "43": 10.77432, - "44": 10.77182, - "45": 10.77167, - "46": 10.75137, - "47": 10.73716, - "48": 10.71898, + "42": 10.77756, + "43": 10.77435, + "44": 10.77189, + "45": 10.77173, + "46": 10.75143, + "47": 10.73718, + "48": 10.71907, "49": 10.73121, - "50": 10.71218 + "50": 10.71224 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 23026300.0, - "2": 22914086.0, - "3": 22773382.0, - "4": 22840872.0, - "5": 22848084.0, - "6": 22810006.0, - "7": 22935596.0, - "8": 22675152.0, - "9": 22822660.0, - "10": 22552656.0, - "11": 22817710.0, - "12": 22705288.0, - "13": 23386302.0, - "14": 23056148.0, - "15": 22781850.0, - "16": 22886906.0, - "17": 22993636.0, - "18": 23058726.0, - "19": 23148372.0, - "20": 22787076.0, - "21": 22981484.0, - "22": 23010492.0, - "23": 22692514.0, - "24": 22925772.0, - "25": 22699682.0, - "26": 23068768.0, - "27": 22865260.0, - "28": 23063292.0, - "29": 23042696.0, - "30": 23008720.0, - "31": 22972018.0, - "32": 22730204.0, - "33": 22804204.0, - "34": 23139172.0, - "35": 22813746.0, - "36": 22758552.0, - "37": 23167014.0, - "38": 23028354.0, - "39": 23048620.0, - "40": 22812828.0, - "41": 23129236.0, - "42": 22752788.0, - "43": 23050796.0, - "44": 22767704.0, - "45": 22909896.0, - "46": 22797186.0, - "47": 22913150.0, - "48": 22896154.0, - "49": 22952004.0, - "50": 22707804.0 + "1": 23026284.0, + "2": 22914140.0, + "3": 22773236.0, + "4": 22840898.0, + "5": 22848020.0, + "6": 22809990.0, + "7": 22935564.0, + "8": 22675156.0, + "9": 22822612.0, + "10": 22552600.0, + "11": 22817702.0, + "12": 22705280.0, + "13": 23386330.0, + "14": 23056232.0, + "15": 22781852.0, + "16": 22886936.0, + "17": 22993724.0, + "18": 23058680.0, + "19": 23148428.0, + "20": 22787088.0, + "21": 22981476.0, + "22": 23010380.0, + "23": 22692468.0, + "24": 22925756.0, + "25": 22699584.0, + "26": 23068772.0, + "27": 22865264.0, + "28": 23063288.0, + "29": 23042648.0, + "30": 23008806.0, + "31": 22972060.0, + "32": 22730272.0, + "33": 22804248.0, + "34": 23139150.0, + "35": 22813676.0, + "36": 22758546.0, + "37": 23167030.0, + "38": 23028400.0, + "39": 23048512.0, + "40": 22812806.0, + "41": 23129286.0, + "42": 22752824.0, + "43": 23050778.0, + "44": 22767736.0, + "45": 22909854.0, + "46": 22797204.0, + "47": 22913040.0, + "48": 22896188.0, + "49": 22951984.0, + "50": 22707788.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.91356, - "3": 2.24295, - "4": 1.92336, - "5": 1.61239, - "6": 1.86835, - "7": 1.45068, - "8": 2.21581, - "9": 2.65865, - "10": 1.77534, - "11": 2.10301, - "12": 1.69138, - "13": 2.18653, - "14": 2.08534, - "15": 1.69653, - "16": 2.02157, - "17": 2.59276, - "18": 1.82518, - "19": 2.03353, - "20": 1.93249, - "21": 2.19679, - "22": 1.74841, - "23": 1.88576, - "24": 1.77505, - "25": 1.98962, - "26": 1.95857, - "27": 1.83598, - "28": 1.79531, - "29": 1.52829, - "30": 1.83218, - "31": 1.73951, - "32": 2.62809, - "33": 1.69551, - "34": 1.51829, - "35": 2.27775, - "36": 2.20222, - "37": 1.6045, - "38": 1.54015, - "39": 2.09868, - "40": 1.6369, - "41": 1.61275, - "42": 2.73991, - "43": 1.57157, - "44": 1.88966, - "45": 2.03006, - "46": 1.84791, - "47": 1.72333, - "48": 2.02252, - "49": 1.69002, - "50": 2.47843 + "2": 5.12069, + "3": 1.9289, + "4": 1.98432, + "5": 1.37098, + "6": 1.3933, + "7": 1.51543, + "8": 2.14314, + "9": 1.97769, + "10": 1.82184, + "11": 1.93202, + "12": 1.63522, + "13": 1.97043, + "14": 2.10776, + "15": 1.34972, + "16": 1.97639, + "17": 2.37643, + "18": 1.73916, + "19": 1.56093, + "20": 1.56061, + "21": 1.89827, + "22": 1.61617, + "23": 1.91279, + "24": 1.39084, + "25": 2.11379, + "26": 1.79649, + "27": 1.63637, + "28": 1.6207, + "29": 1.43216, + "30": 1.65715, + "31": 1.54835, + "32": 2.83639, + "33": 1.51551, + "34": 1.60488, + "35": 2.01577, + "36": 2.0697, + "37": 1.83256, + "38": 1.37463, + "39": 2.04055, + "40": 1.90678, + "41": 1.73592, + "42": 2.05084, + "43": 1.7771, + "44": 1.54443, + "45": 2.13217, + "46": 1.75515, + "47": 1.66892, + "48": 2.02237, + "49": 1.63104, + "50": 1.71892 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json index e706b8f5b63..f11ad9ca3be 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, - "3": 10.94862, + "1": 10.95177, + "2": 10.94867, + "3": 10.94863, "4": 10.94969, - "5": 10.94753, - "6": 10.94472, + "5": 10.94755, + "6": 10.94479, "7": 10.9465, - "8": 10.94472, - "9": 10.94186, - "10": 10.94475, + "8": 10.94479, + "9": 10.94191, + "10": 10.94479, "11": 10.94106, - "12": 10.93297, + "12": 10.93291, "13": 10.92308, - "14": 10.92593, - "15": 10.89969, - "16": 10.89084, - "17": 10.89814, - "18": 10.88337, - "19": 10.88981, - "20": 10.79731, - "21": 10.79299, - "22": 10.77996, - "23": 10.76964, - "24": 10.74573, - "25": 10.73536, - "26": 10.72512, - "27": 10.67795, - "28": 10.62002, - "29": 10.59158, - "30": 10.56328, - "31": 10.56514, - "32": 10.54392, - "33": 10.50914, - "34": 10.48321, - "35": 10.46994, - "36": 10.44683, - "37": 10.42648, - "38": 10.43048, - "39": 10.39271, - "40": 10.38041, - "41": 10.35136, - "42": 10.3275, + "14": 10.92589, + "15": 10.89963, + "16": 10.89081, + "17": 10.89822, + "18": 10.88344, + "19": 10.88987, + "20": 10.7974, + "21": 10.79296, + "22": 10.78008, + "23": 10.76968, + "24": 10.74582, + "25": 10.73534, + "26": 10.72506, + "27": 10.67793, + "28": 10.62001, + "29": 10.59163, + "30": 10.56335, + "31": 10.56519, + "32": 10.544, + "33": 10.50913, + "34": 10.48323, + "35": 10.4699, + "36": 10.44694, + "37": 10.42657, + "38": 10.43051, + "39": 10.39275, + "40": 10.3804, + "41": 10.35141, + "42": 10.32755, "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24783, - "47": 10.2438, + "44": 10.28348, + "45": 10.29608, + "46": 10.24785, + "47": 10.24385, "48": 10.19393, - "49": 10.19673, - "50": 10.18924, - "51": 10.19745, + "49": 10.19678, + "50": 10.18927, + "51": 10.19748, "52": 10.15223, - "53": 10.15768, - "54": 10.12071, - "55": 10.09673, - "56": 10.12108, - "57": 10.11169, - "58": 10.12097, - "59": 10.06243, - "60": 10.09231, - "61": 10.04173, - "62": 10.00957, - "63": 10.07951, - "64": 10.03237, - "65": 10.00321, - "66": 10.03953, - "67": 10.01958, - "68": 9.9857, - "69": 10.00195, + "53": 10.15775, + "54": 10.12079, + "55": 10.09676, + "56": 10.1211, + "57": 10.11179, + "58": 10.12101, + "59": 10.06249, + "60": 10.0924, + "61": 10.04177, + "62": 10.00959, + "63": 10.0795, + "64": 10.03239, + "65": 10.00322, + "66": 10.03956, + "67": 10.01964, + "68": 9.98577, + "69": 10.00204, "70": 9.98144, - "71": 10.00456, - "72": 9.99623, - "73": 9.9876, - "74": 9.97288, - "75": 9.93232, - "76": 9.96721, - "77": 9.96946, - "78": 9.92083, - "79": 9.91736, - "80": 9.93738, - "81": 9.95721, - "82": 9.89591, - "83": 9.86219, - "84": 9.80485, - "85": 9.78642, - "86": 9.89557, - "87": 9.90872, + "71": 10.00455, + "72": 9.99626, + "73": 9.98765, + "74": 9.97294, + "75": 9.93237, + "76": 9.9672, + "77": 9.96954, + "78": 9.92086, + "79": 9.9174, + "80": 9.93744, + "81": 9.95731, + "82": 9.89594, + "83": 9.86223, + "84": 9.80486, + "85": 9.78646, + "86": 9.89559, + "87": 9.90877, "88": 9.88858, - "89": 9.83328, - "90": 9.82422, - "91": 9.8393, - "92": 9.82245, - "93": 9.75866, - "94": 9.83495, - "95": 9.82146, - "96": 9.81157, - "97": 9.75625, - "98": 9.78244, - "99": 9.82335, - "100": 9.71583 + "89": 9.8333, + "90": 9.82427, + "91": 9.83932, + "92": 9.82252, + "93": 9.75876, + "94": 9.83501, + "95": 9.82152, + "96": 9.81164, + "97": 9.75629, + "98": 9.78246, + "99": 9.82339, + "100": 9.71587 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1733.0, - "2": 1654.0, - "3": 1807.0, - "4": 1633.0, - "5": 1831.0, - "6": 1678.0, - "7": 1895.0, - "8": 1671.0, - "9": 1673.0, - "10": 1708.0, - "11": 1553.0, - "12": 1673.0, - "13": 1817.0, - "14": 1834.0, - "15": 1646.0, - "16": 1768.0, - "17": 1815.0, - "18": 1755.0, - "19": 1673.0, - "20": 1652.0, - "21": 1774.0, - "22": 1666.0, - "23": 1742.0, - "24": 1813.0, - "25": 1613.0, - "26": 1757.0, - "27": 1933.0, - "28": 1878.0, - "29": 1897.0, - "30": 1773.0, - "31": 2012.0, - "32": 1945.0, - "33": 1954.0, - "34": 2032.0, - "35": 2079.0, - "36": 1910.0, - "37": 2215.0, - "38": 2091.0, - "39": 2118.0, - "40": 2207.0, - "41": 2215.0, - "42": 2146.0, - "43": 2268.0, - "44": 2277.0, - "45": 2471.0, - "46": 2361.0, - "47": 2478.0, - "48": 2516.0, - "49": 2742.0, - "50": 2642.0, - "51": 2514.0, - "52": 2737.0, - "53": 2633.0, - "54": 2812.0, - "55": 2551.0, - "56": 2785.0, - "57": 2305.0, - "58": 3578.0, - "59": 2929.0, - "60": 2969.0, - "61": 2767.0, - "62": 3207.0, - "63": 3261.0, - "64": 3564.0, - "65": 2658.0, - "66": 3028.0, - "67": 3825.0, - "68": 3302.0, - "69": 2965.0, - "70": 3273.0, - "71": 3178.0, - "72": 2916.0, - "73": 3501.0, - "74": 3278.0, - "75": 3294.0, - "76": 3359.0, - "77": 3682.0, - "78": 3230.0, - "79": 3223.0, - "80": 2932.0, - "81": 3215.0, - "82": 2894.0, - "83": 2944.0, - "84": 3031.0, - "85": 2667.0, - "86": 3211.0, - "87": 2910.0, - "88": 3105.0, - "89": 2985.0, - "90": 3930.0, - "91": 2757.0, - "92": 3004.0, - "93": 2976.0, - "94": 3091.0, - "95": 3323.0, - "96": 3464.0, - "97": 3570.0, - "98": 3215.0, - "99": 3459.0, - "100": 3386.0 + "1": 1720.0, + "2": 1610.0, + "3": 1759.0, + "4": 1672.0, + "5": 1790.0, + "6": 1645.0, + "7": 1951.0, + "8": 1673.0, + "9": 1638.0, + "10": 1711.0, + "11": 1665.0, + "12": 1693.0, + "13": 1756.0, + "14": 1856.0, + "15": 1588.0, + "16": 1726.0, + "17": 1789.0, + "18": 1794.0, + "19": 1682.0, + "20": 1650.0, + "21": 1797.0, + "22": 1651.0, + "23": 1734.0, + "24": 1800.0, + "25": 1706.0, + "26": 1766.0, + "27": 1833.0, + "28": 1796.0, + "29": 1794.0, + "30": 1818.0, + "31": 1985.0, + "32": 1935.0, + "33": 1949.0, + "34": 2021.0, + "35": 2052.0, + "36": 2020.0, + "37": 2106.0, + "38": 2035.0, + "39": 2189.0, + "40": 2178.0, + "41": 2319.0, + "42": 2030.0, + "43": 2423.0, + "44": 2211.0, + "45": 2387.0, + "46": 2376.0, + "47": 2480.0, + "48": 2587.0, + "49": 2711.0, + "50": 2618.0, + "51": 2475.0, + "52": 2747.0, + "53": 2656.0, + "54": 2815.0, + "55": 2536.0, + "56": 2716.0, + "57": 2263.0, + "58": 3635.0, + "59": 3047.0, + "60": 3031.0, + "61": 2806.0, + "62": 3246.0, + "63": 3289.0, + "64": 3593.0, + "65": 2540.0, + "66": 2944.0, + "67": 3836.0, + "68": 3485.0, + "69": 2941.0, + "70": 3355.0, + "71": 3196.0, + "72": 2909.0, + "73": 3498.0, + "74": 3224.0, + "75": 3120.0, + "76": 3269.0, + "77": 3648.0, + "78": 3293.0, + "79": 3169.0, + "80": 3022.0, + "81": 3305.0, + "82": 2934.0, + "83": 3071.0, + "84": 3060.0, + "85": 2653.0, + "86": 3079.0, + "87": 2941.0, + "88": 3158.0, + "89": 3045.0, + "90": 3795.0, + "91": 2844.0, + "92": 2887.0, + "93": 2965.0, + "94": 2974.0, + "95": 3328.0, + "96": 3516.0, + "97": 3595.0, + "98": 3304.0, + "99": 3431.0, + "100": 3362.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.80466, - "3": 0.18387, - "4": 0.18412, - "5": 0.18453, - "6": 0.18603, - "7": 0.1839, - "8": 0.18381, - "9": 0.18608, - "10": 0.18453, - "11": 0.18388, - "12": 0.18211, - "13": 0.18714, - "14": 0.18529, - "15": 0.1852, - "16": 0.1845, - "17": 0.18358, - "18": 0.18631, - "19": 0.18375, - "20": 0.18266, - "21": 0.18293, - "22": 0.18089, - "23": 0.18406, - "24": 0.18205, - "25": 0.18155, - "26": 0.18385, - "27": 0.18228, - "28": 0.18249, - "29": 0.1837, - "30": 0.18115, - "31": 0.17933, - "32": 0.18245, - "33": 0.18048, - "34": 0.17919, - "35": 0.18161, - "36": 0.18044, - "37": 0.18038, - "38": 0.18397, - "39": 0.18404, - "40": 0.1804, - "41": 0.18276, - "42": 0.18045, - "43": 0.18052, - "44": 0.18131, - "45": 0.18034, - "46": 0.18231, - "47": 0.18155, - "48": 0.17974, - "49": 0.18214, - "50": 0.1806, - "51": 0.20772, - "52": 0.79964, - "53": 0.18042, - "54": 0.18164, - "55": 0.18053, - "56": 0.18219, - "57": 0.17923, - "58": 0.18048, - "59": 0.18007, - "60": 0.18026, - "61": 0.17951, - "62": 0.17958, - "63": 0.18152, - "64": 0.17994, - "65": 0.17811, - "66": 0.17964, - "67": 0.17856, - "68": 0.18123, - "69": 0.18032, - "70": 0.17993, - "71": 0.17927, - "72": 0.17812, - "73": 0.18081, - "74": 0.18039, - "75": 0.18345, - "76": 0.17951, - "77": 0.17747, - "78": 0.17942, - "79": 0.17913, - "80": 0.18271, - "81": 0.17983, - "82": 0.17912, - "83": 0.18108, - "84": 0.18144, - "85": 0.18107, - "86": 0.18062, - "87": 0.17977, - "88": 0.18167, - "89": 0.18068, - "90": 0.18019, - "91": 0.18019, - "92": 0.17872, - "93": 0.17945, - "94": 0.17979, - "95": 0.17803, - "96": 0.18154, - "97": 0.17801, - "98": 0.18046, - "99": 0.18003, - "100": 0.22411 + "2": 5.24682, + "3": 0.19742, + "4": 0.19537, + "5": 0.19477, + "6": 0.19373, + "7": 0.19347, + "8": 0.19628, + "9": 0.19374, + "10": 0.19525, + "11": 0.19375, + "12": 0.19481, + "13": 0.19422, + "14": 0.19545, + "15": 0.19776, + "16": 0.19453, + "17": 0.19366, + "18": 0.19518, + "19": 0.19624, + "20": 0.19499, + "21": 0.19553, + "22": 0.19556, + "23": 0.19334, + "24": 0.19426, + "25": 0.19282, + "26": 0.19994, + "27": 0.19434, + "28": 0.19531, + "29": 0.19551, + "30": 0.21582, + "31": 0.19337, + "32": 0.19344, + "33": 0.19501, + "34": 0.19372, + "35": 0.19446, + "36": 0.19528, + "37": 0.19536, + "38": 0.19425, + "39": 0.19717, + "40": 0.19664, + "41": 0.19698, + "42": 0.37111, + "43": 0.34002, + "44": 0.39839, + "45": 0.20914, + "46": 0.21004, + "47": 0.20258, + "48": 0.1992, + "49": 0.19504, + "50": 0.1962, + "51": 0.37871, + "52": 0.82003, + "53": 0.1955, + "54": 0.19439, + "55": 0.19441, + "56": 0.19344, + "57": 0.19385, + "58": 0.1935, + "59": 0.19298, + "60": 0.19334, + "61": 0.19425, + "62": 0.19303, + "63": 0.19336, + "64": 0.19594, + "65": 0.19434, + "66": 0.19214, + "67": 0.1929, + "68": 0.19409, + "69": 0.19177, + "70": 0.19012, + "71": 0.19241, + "72": 0.19195, + "73": 0.19138, + "74": 0.19104, + "75": 0.19056, + "76": 0.19134, + "77": 0.18898, + "78": 0.19177, + "79": 0.19223, + "80": 0.1902, + "81": 0.19106, + "82": 0.18796, + "83": 0.19008, + "84": 0.19281, + "85": 0.19128, + "86": 0.18878, + "87": 0.19079, + "88": 0.19005, + "89": 0.18813, + "90": 0.19111, + "91": 0.19103, + "92": 0.19145, + "93": 0.19072, + "94": 0.18852, + "95": 0.18916, + "96": 0.19335, + "97": 0.18875, + "98": 0.18943, + "99": 0.19374, + "100": 0.19183 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json index 61b91e9f7fa..672923c202c 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92866, - "4": 10.9288, - "5": 10.93088, - "6": 10.92771, - "7": 10.92336, - "8": 10.92159, - "9": 10.92031, + "2": 10.92832, + "3": 10.92864, + "4": 10.92888, + "5": 10.93089, + "6": 10.92772, + "7": 10.92341, + "8": 10.92161, + "9": 10.92027, "10": 10.91964, - "11": 10.91027, - "12": 10.91814, - "13": 10.91523, - "14": 10.89825, - "15": 10.88042, - "16": 10.86841, + "11": 10.91024, + "12": 10.91826, + "13": 10.91526, + "14": 10.89832, + "15": 10.88051, + "16": 10.8684, "17": 10.86882, - "18": 10.86258, - "19": 10.86185, - "20": 10.77348, - "21": 10.75516, - "22": 10.75521, - "23": 10.74508, - "24": 10.71106, - "25": 10.71082, - "26": 10.69968, - "27": 10.65192, - "28": 10.58249, - "29": 10.55636, - "30": 10.54863, - "31": 10.52499, - "32": 10.51761, - "33": 10.48144, - "34": 10.451, - "35": 10.44994, - "36": 10.43486, - "37": 10.39411, - "38": 10.39546, - "39": 10.36016, - "40": 10.35798, - "41": 10.32782, - "42": 10.30704, - "43": 10.28206, - "44": 10.25604, - "45": 10.27039, - "46": 10.23404, - "47": 10.21882, + "18": 10.86249, + "19": 10.86187, + "20": 10.77339, + "21": 10.75518, + "22": 10.75531, + "23": 10.74509, + "24": 10.71112, + "25": 10.71086, + "26": 10.69962, + "27": 10.65199, + "28": 10.58247, + "29": 10.55645, + "30": 10.54865, + "31": 10.52506, + "32": 10.51769, + "33": 10.48149, + "34": 10.45111, + "35": 10.44997, + "36": 10.43493, + "37": 10.3941, + "38": 10.39549, + "39": 10.36015, + "40": 10.35806, + "41": 10.32785, + "42": 10.30705, + "43": 10.28217, + "44": 10.25607, + "45": 10.27045, + "46": 10.23413, + "47": 10.21893, "48": 10.17023, - "49": 10.1759, - "50": 10.17855, - "51": 10.19228, - "52": 10.13723, - "53": 10.13552, - "54": 10.10521, - "55": 10.08055, - "56": 10.11116, - "57": 10.09599, - "58": 10.10931, - "59": 10.05623, - "60": 10.07399, - "61": 10.02775, - "62": 10.0003, - "63": 10.07322, - "64": 10.03695, - "65": 10.00341, - "66": 10.03038, - "67": 10.0072, - "68": 9.97019, - "69": 9.9915, - "70": 9.97453, - "71": 10.00065, - "72": 9.97779, - "73": 9.97166, - "74": 9.95604, - "75": 9.93149, - "76": 9.96344, - "77": 9.95639, - "78": 9.90552, - "79": 9.91218, - "80": 9.92898, - "81": 9.95655, - "82": 9.89202, - "83": 9.85589, - "84": 9.78695, - "85": 9.78572, - "86": 9.88392, - "87": 9.91028, - "88": 9.88719, - "89": 9.81819, - "90": 9.81249, - "91": 9.82779, - "92": 9.81659, - "93": 9.74998, - "94": 9.8309, - "95": 9.82648, - "96": 9.80702, - "97": 9.74393, - "98": 9.77873, - "99": 9.82058, - "100": 9.71398 + "49": 10.17594, + "50": 10.17859, + "51": 10.19235, + "52": 10.13725, + "53": 10.13558, + "54": 10.10524, + "55": 10.08058, + "56": 10.11123, + "57": 10.09601, + "58": 10.10934, + "59": 10.05632, + "60": 10.07401, + "61": 10.02777, + "62": 10.00036, + "63": 10.07317, + "64": 10.03696, + "65": 10.00345, + "66": 10.0304, + "67": 10.00724, + "68": 9.97024, + "69": 9.99153, + "70": 9.9746, + "71": 10.00066, + "72": 9.97785, + "73": 9.9717, + "74": 9.95607, + "75": 9.93157, + "76": 9.96345, + "77": 9.95644, + "78": 9.90557, + "79": 9.9122, + "80": 9.92906, + "81": 9.95666, + "82": 9.89201, + "83": 9.85599, + "84": 9.78701, + "85": 9.7858, + "86": 9.88398, + "87": 9.91034, + "88": 9.88728, + "89": 9.81818, + "90": 9.81251, + "91": 9.82784, + "92": 9.81667, + "93": 9.75004, + "94": 9.83094, + "95": 9.82651, + "96": 9.80705, + "97": 9.74396, + "98": 9.77876, + "99": 9.82056, + "100": 9.71405 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1731.0, - "2": 1691.0, - "3": 1669.0, - "4": 1581.0, - "5": 1644.0, - "6": 1685.0, - "7": 1815.0, - "8": 1709.0, - "9": 1781.0, - "10": 1748.0, - "11": 1630.0, - "12": 1732.0, - "13": 1742.0, - "14": 1815.0, - "15": 1549.0, - "16": 1654.0, - "17": 1749.0, - "18": 1742.0, - "19": 1686.0, - "20": 1643.0, - "21": 1695.0, - "22": 1733.0, - "23": 1658.0, - "24": 1781.0, - "25": 1666.0, - "26": 1789.0, - "27": 1777.0, - "28": 1803.0, - "29": 1831.0, - "30": 1831.0, - "31": 2000.0, - "32": 1953.0, - "33": 1927.0, - "34": 2012.0, - "35": 2071.0, - "36": 1876.0, - "37": 2178.0, - "38": 2133.0, - "39": 2155.0, - "40": 2357.0, - "41": 2339.0, - "42": 1887.0, - "43": 2351.0, - "44": 2221.0, - "45": 2475.0, - "46": 2480.0, - "47": 2497.0, + "1": 1684.0, + "2": 1628.0, + "3": 1753.0, + "4": 1661.0, + "5": 1709.0, + "6": 1619.0, + "7": 1824.0, + "8": 1706.0, + "9": 1707.0, + "10": 1648.0, + "11": 1619.0, + "12": 1773.0, + "13": 1782.0, + "14": 1928.0, + "15": 1597.0, + "16": 1623.0, + "17": 1698.0, + "18": 1750.0, + "19": 1626.0, + "20": 1618.0, + "21": 1771.0, + "22": 1706.0, + "23": 1626.0, + "24": 1718.0, + "25": 1588.0, + "26": 1757.0, + "27": 1832.0, + "28": 1744.0, + "29": 1941.0, + "30": 1853.0, + "31": 1889.0, + "32": 1871.0, + "33": 1977.0, + "34": 2042.0, + "35": 2009.0, + "36": 1943.0, + "37": 2226.0, + "38": 2146.0, + "39": 2107.0, + "40": 2290.0, + "41": 2277.0, + "42": 1917.0, + "43": 2428.0, + "44": 2234.0, + "45": 2588.0, + "46": 2488.0, + "47": 2491.0, "48": 2601.0, - "49": 2909.0, - "50": 2638.0, - "51": 2495.0, - "52": 2790.0, - "53": 2729.0, - "54": 2777.0, - "55": 2626.0, - "56": 2743.0, - "57": 2177.0, - "58": 3682.0, - "59": 2899.0, - "60": 2899.0, - "61": 2826.0, - "62": 3369.0, - "63": 3377.0, - "64": 3714.0, - "65": 2750.0, - "66": 3201.0, - "67": 3778.0, - "68": 3542.0, - "69": 3015.0, - "70": 3264.0, - "71": 3129.0, - "72": 3054.0, - "73": 3365.0, - "74": 3234.0, - "75": 3345.0, - "76": 3367.0, - "77": 3958.0, - "78": 3329.0, - "79": 3194.0, - "80": 2951.0, - "81": 3592.0, - "82": 2943.0, - "83": 3073.0, - "84": 3037.0, - "85": 2541.0, - "86": 3023.0, - "87": 2916.0, - "88": 3027.0, - "89": 3180.0, - "90": 3778.0, - "91": 3014.0, - "92": 2726.0, - "93": 3051.0, - "94": 3000.0, - "95": 3189.0, - "96": 3382.0, - "97": 3490.0, - "98": 3305.0, - "99": 3298.0, - "100": 3311.0 + "49": 2860.0, + "50": 2622.0, + "51": 2566.0, + "52": 2713.0, + "53": 2694.0, + "54": 2837.0, + "55": 2660.0, + "56": 2691.0, + "57": 2100.0, + "58": 3764.0, + "59": 2897.0, + "60": 3048.0, + "61": 2813.0, + "62": 3200.0, + "63": 3426.0, + "64": 3603.0, + "65": 2607.0, + "66": 3090.0, + "67": 3762.0, + "68": 3476.0, + "69": 3189.0, + "70": 3310.0, + "71": 3107.0, + "72": 3040.0, + "73": 3459.0, + "74": 3262.0, + "75": 3167.0, + "76": 3312.0, + "77": 3909.0, + "78": 3290.0, + "79": 3200.0, + "80": 3133.0, + "81": 3597.0, + "82": 2947.0, + "83": 3172.0, + "84": 2917.0, + "85": 2518.0, + "86": 3140.0, + "87": 2832.0, + "88": 3076.0, + "89": 3067.0, + "90": 3800.0, + "91": 3056.0, + "92": 2804.0, + "93": 3105.0, + "94": 3146.0, + "95": 3205.0, + "96": 3332.0, + "97": 3391.0, + "98": 3361.0, + "99": 3318.0, + "100": 3245.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.06977, - "3": 2.18555, - "4": 2.0122, - "5": 1.5422, - "6": 1.88019, - "7": 2.20797, - "8": 2.71542, - "9": 2.10583, - "10": 2.53978, - "11": 2.52922, - "12": 1.72555, - "13": 1.99992, - "14": 2.65834, - "15": 1.93394, - "16": 2.06383, - "17": 2.36017, - "18": 2.0535, - "19": 1.90883, - "20": 2.08271, - "21": 2.25995, - "22": 1.92834, - "23": 1.74054, - "24": 1.70254, - "25": 2.54029, - "26": 2.39901, - "27": 1.99731, - "28": 1.98739, - "29": 1.50029, - "30": 1.96645, - "31": 2.15154, - "32": 2.56678, - "33": 1.99054, - "34": 2.13667, - "35": 2.64511, - "36": 2.09883, - "37": 1.91001, - "38": 1.5396, - "39": 2.46548, - "40": 1.9233, - "41": 2.28732, - "42": 2.35301, - "43": 1.65789, - "44": 1.56099, - "45": 2.05719, - "46": 1.70137, - "47": 1.68981, - "48": 1.92915, - "49": 1.66933, - "50": 1.88296, - "51": 1.23599, - "52": 2.07863, - "53": 1.86058, - "54": 1.69935, - "55": 1.78663, - "56": 2.13775, - "57": 1.19895, - "58": 2.12168, - "59": 1.45477, - "60": 1.47497, - "61": 1.904, - "62": 2.27901, - "63": 1.59531, - "64": 2.3399, - "65": 2.91356, - "66": 2.56806, - "67": 2.2204, - "68": 2.10613, - "69": 1.99613, - "70": 2.2961, - "71": 2.4294, - "72": 2.12526, - "73": 2.02606, - "74": 1.36499, - "75": 2.07417, - "76": 1.94247, - "77": 1.9907, - "78": 1.86404, - "79": 2.14268, - "80": 1.72401, - "81": 2.07887, - "82": 1.62928, - "83": 1.88239, - "84": 1.52229, - "85": 1.61961, - "86": 1.78662, - "87": 2.11775, - "88": 1.8844, - "89": 1.71247, - "90": 1.66331, - "91": 1.76112, - "92": 1.83053, - "93": 1.41142, - "94": 1.51544, - "95": 1.81539, - "96": 1.68572, - "97": 1.69888, - "98": 1.88079, - "99": 2.63883, - "100": 1.59243 + "2": 4.84356, + "3": 2.10378, + "4": 1.82387, + "5": 1.6608, + "6": 2.04969, + "7": 1.70108, + "8": 2.39498, + "9": 2.37876, + "10": 1.9171, + "11": 1.99861, + "12": 1.5078, + "13": 1.87241, + "14": 1.8319, + "15": 1.6103, + "16": 1.75308, + "17": 2.45481, + "18": 1.86227, + "19": 2.05276, + "20": 1.13922, + "21": 1.88636, + "22": 1.36599, + "23": 1.86739, + "24": 1.51077, + "25": 2.15755, + "26": 2.47908, + "27": 1.65764, + "28": 1.57282, + "29": 1.35662, + "30": 1.7436, + "31": 1.45309, + "32": 2.46891, + "33": 1.49797, + "34": 1.16963, + "35": 1.87516, + "36": 2.04065, + "37": 1.76393, + "38": 1.506, + "39": 1.84247, + "40": 1.71372, + "41": 1.84742, + "42": 1.70379, + "43": 1.75669, + "44": 1.5525, + "45": 2.06469, + "46": 1.4035, + "47": 1.88623, + "48": 2.191, + "49": 1.22899, + "50": 1.77852, + "51": 1.27485, + "52": 1.51136, + "53": 1.85535, + "54": 1.75792, + "55": 1.77536, + "56": 2.50275, + "57": 1.28222, + "58": 1.89316, + "59": 1.82598, + "60": 1.62966, + "61": 2.14771, + "62": 2.08371, + "63": 1.63791, + "64": 1.63849, + "65": 1.5963, + "66": 2.12075, + "67": 1.7796, + "68": 1.30268, + "69": 1.79236, + "70": 1.87103, + "71": 1.85465, + "72": 1.53627, + "73": 1.77275, + "74": 1.44437, + "75": 1.20086, + "76": 1.51223, + "77": 1.32181, + "78": 1.41545, + "79": 1.58278, + "80": 1.46161, + "81": 1.62663, + "82": 1.85105, + "83": 1.58688, + "84": 1.26104, + "85": 1.35199, + "86": 1.46799, + "87": 2.09369, + "88": 1.55899, + "89": 1.71657, + "90": 1.63632, + "91": 1.72679, + "92": 1.76589, + "93": 1.43944, + "94": 2.04429, + "95": 1.71654, + "96": 1.55004, + "97": 1.72909, + "98": 2.04633, + "99": 2.75055, + "100": 1.72548 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json index e02afd01b1f..8b82283f910 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, - "3": 10.94862, + "1": 10.95177, + "2": 10.94867, + "3": 10.94863, "4": 10.94969, - "5": 10.94753, - "6": 10.94472, + "5": 10.94755, + "6": 10.94479, "7": 10.9465, - "8": 10.94472, - "9": 10.94186, - "10": 10.94475, + "8": 10.94479, + "9": 10.94191, + "10": 10.94479, "11": 10.94106, - "12": 10.93297, + "12": 10.93291, "13": 10.92308, - "14": 10.92593, - "15": 10.89969, - "16": 10.89084, - "17": 10.89814, - "18": 10.88337, - "19": 10.88981, - "20": 10.79731, - "21": 10.79299, - "22": 10.77996, - "23": 10.76964, - "24": 10.74573, - "25": 10.73536, - "26": 10.72512, - "27": 10.67795, - "28": 10.62002, - "29": 10.59158, - "30": 10.56328, - "31": 10.56514, - "32": 10.54392, - "33": 10.50914, - "34": 10.48321, - "35": 10.46994, - "36": 10.44683, - "37": 10.42648, - "38": 10.43048, - "39": 10.39271, - "40": 10.38041, - "41": 10.35136, - "42": 10.3275, + "14": 10.92589, + "15": 10.89963, + "16": 10.89081, + "17": 10.89822, + "18": 10.88344, + "19": 10.88987, + "20": 10.7974, + "21": 10.79296, + "22": 10.78008, + "23": 10.76968, + "24": 10.74582, + "25": 10.73534, + "26": 10.72506, + "27": 10.67793, + "28": 10.62001, + "29": 10.59163, + "30": 10.56335, + "31": 10.56519, + "32": 10.544, + "33": 10.50913, + "34": 10.48323, + "35": 10.4699, + "36": 10.44694, + "37": 10.42657, + "38": 10.43051, + "39": 10.39275, + "40": 10.3804, + "41": 10.35141, + "42": 10.32755, "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24783, - "47": 10.2438, + "44": 10.28348, + "45": 10.29608, + "46": 10.24785, + "47": 10.24385, "48": 10.19393, - "49": 10.19673, - "50": 10.18924, - "51": 10.19745, + "49": 10.19678, + "50": 10.18927, + "51": 10.19748, "52": 10.15223, - "53": 10.15768, - "54": 10.12071, - "55": 10.09673, - "56": 10.12108, - "57": 10.11169, - "58": 10.12097, - "59": 10.06243, - "60": 10.09231, - "61": 10.04173, - "62": 10.00957, - "63": 10.07951, - "64": 10.03237, - "65": 10.00321, - "66": 10.03953, - "67": 10.01958, - "68": 9.9857, - "69": 10.00195, + "53": 10.15775, + "54": 10.12079, + "55": 10.09676, + "56": 10.1211, + "57": 10.11179, + "58": 10.12101, + "59": 10.06249, + "60": 10.0924, + "61": 10.04177, + "62": 10.00959, + "63": 10.0795, + "64": 10.03239, + "65": 10.00322, + "66": 10.03956, + "67": 10.01964, + "68": 9.98577, + "69": 10.00204, "70": 9.98144, - "71": 10.00456, - "72": 9.99623, - "73": 9.9876, - "74": 9.97288, - "75": 9.93232, - "76": 9.96721, - "77": 9.96946, - "78": 9.92083, - "79": 9.91736, - "80": 9.93738, - "81": 9.95721, - "82": 9.89591, - "83": 9.86219, - "84": 9.80485, - "85": 9.78642, - "86": 9.89557, - "87": 9.90872, + "71": 10.00455, + "72": 9.99626, + "73": 9.98765, + "74": 9.97294, + "75": 9.93237, + "76": 9.9672, + "77": 9.96954, + "78": 9.92086, + "79": 9.9174, + "80": 9.93744, + "81": 9.95731, + "82": 9.89594, + "83": 9.86223, + "84": 9.80486, + "85": 9.78646, + "86": 9.89559, + "87": 9.90877, "88": 9.88858, - "89": 9.83328, - "90": 9.82422, - "91": 9.8393, - "92": 9.82245, - "93": 9.75866, - "94": 9.83495, - "95": 9.82146, - "96": 9.81157, - "97": 9.75625, - "98": 9.78244, - "99": 9.82335, - "100": 9.71583 + "89": 9.8333, + "90": 9.82427, + "91": 9.83932, + "92": 9.82252, + "93": 9.75876, + "94": 9.83501, + "95": 9.82152, + "96": 9.81164, + "97": 9.75629, + "98": 9.78246, + "99": 9.82339, + "100": 9.71587 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1733.0, - "2": 1654.0, - "3": 1807.0, - "4": 1633.0, - "5": 1831.0, - "6": 1678.0, - "7": 1895.0, - "8": 1671.0, - "9": 1673.0, - "10": 1708.0, - "11": 1553.0, - "12": 1673.0, - "13": 1817.0, - "14": 1834.0, - "15": 1646.0, - "16": 1768.0, - "17": 1815.0, - "18": 1755.0, - "19": 1673.0, - "20": 1652.0, - "21": 1774.0, - "22": 1666.0, - "23": 1742.0, - "24": 1813.0, - "25": 1613.0, - "26": 1757.0, - "27": 1933.0, - "28": 1878.0, - "29": 1897.0, - "30": 1773.0, - "31": 2012.0, - "32": 1945.0, - "33": 1954.0, - "34": 2032.0, - "35": 2079.0, - "36": 1910.0, - "37": 2215.0, - "38": 2091.0, - "39": 2118.0, - "40": 2207.0, - "41": 2215.0, - "42": 2146.0, - "43": 2268.0, - "44": 2277.0, - "45": 2471.0, - "46": 2361.0, - "47": 2478.0, - "48": 2516.0, - "49": 2742.0, - "50": 2642.0, - "51": 2514.0, - "52": 2737.0, - "53": 2633.0, - "54": 2812.0, - "55": 2551.0, - "56": 2785.0, - "57": 2305.0, - "58": 3578.0, - "59": 2929.0, - "60": 2969.0, - "61": 2767.0, - "62": 3207.0, - "63": 3261.0, - "64": 3564.0, - "65": 2658.0, - "66": 3028.0, - "67": 3825.0, - "68": 3302.0, - "69": 2965.0, - "70": 3273.0, - "71": 3178.0, - "72": 2916.0, - "73": 3501.0, - "74": 3278.0, - "75": 3294.0, - "76": 3359.0, - "77": 3682.0, - "78": 3230.0, - "79": 3223.0, - "80": 2932.0, - "81": 3215.0, - "82": 2894.0, - "83": 2944.0, - "84": 3031.0, - "85": 2667.0, - "86": 3211.0, - "87": 2910.0, - "88": 3105.0, - "89": 2985.0, - "90": 3930.0, - "91": 2757.0, - "92": 3004.0, - "93": 2976.0, - "94": 3091.0, - "95": 3323.0, - "96": 3464.0, - "97": 3570.0, - "98": 3215.0, - "99": 3459.0, - "100": 3386.0 + "1": 1720.0, + "2": 1610.0, + "3": 1759.0, + "4": 1672.0, + "5": 1790.0, + "6": 1645.0, + "7": 1951.0, + "8": 1673.0, + "9": 1638.0, + "10": 1711.0, + "11": 1665.0, + "12": 1693.0, + "13": 1756.0, + "14": 1856.0, + "15": 1588.0, + "16": 1726.0, + "17": 1789.0, + "18": 1794.0, + "19": 1682.0, + "20": 1650.0, + "21": 1797.0, + "22": 1651.0, + "23": 1734.0, + "24": 1800.0, + "25": 1706.0, + "26": 1766.0, + "27": 1833.0, + "28": 1796.0, + "29": 1794.0, + "30": 1818.0, + "31": 1985.0, + "32": 1935.0, + "33": 1949.0, + "34": 2021.0, + "35": 2052.0, + "36": 2020.0, + "37": 2106.0, + "38": 2035.0, + "39": 2189.0, + "40": 2178.0, + "41": 2319.0, + "42": 2030.0, + "43": 2423.0, + "44": 2211.0, + "45": 2387.0, + "46": 2376.0, + "47": 2480.0, + "48": 2587.0, + "49": 2711.0, + "50": 2618.0, + "51": 2475.0, + "52": 2747.0, + "53": 2656.0, + "54": 2815.0, + "55": 2536.0, + "56": 2716.0, + "57": 2263.0, + "58": 3635.0, + "59": 3047.0, + "60": 3031.0, + "61": 2806.0, + "62": 3246.0, + "63": 3289.0, + "64": 3593.0, + "65": 2540.0, + "66": 2944.0, + "67": 3836.0, + "68": 3485.0, + "69": 2941.0, + "70": 3355.0, + "71": 3196.0, + "72": 2909.0, + "73": 3498.0, + "74": 3224.0, + "75": 3120.0, + "76": 3269.0, + "77": 3648.0, + "78": 3293.0, + "79": 3169.0, + "80": 3022.0, + "81": 3305.0, + "82": 2934.0, + "83": 3071.0, + "84": 3060.0, + "85": 2653.0, + "86": 3079.0, + "87": 2941.0, + "88": 3158.0, + "89": 3045.0, + "90": 3795.0, + "91": 2844.0, + "92": 2887.0, + "93": 2965.0, + "94": 2974.0, + "95": 3328.0, + "96": 3516.0, + "97": 3595.0, + "98": 3304.0, + "99": 3431.0, + "100": 3362.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.8204, - "3": 0.1796, - "4": 0.17955, - "5": 0.17744, - "6": 0.17714, - "7": 0.1765, - "8": 0.17799, - "9": 0.176, - "10": 0.1772, - "11": 0.17674, - "12": 0.17776, - "13": 0.17897, - "14": 0.1768, - "15": 0.17847, - "16": 0.17939, - "17": 0.17937, - "18": 0.17718, - "19": 0.17568, - "20": 0.17745, - "21": 0.17946, - "22": 0.17801, - "23": 0.17828, - "24": 0.178, - "25": 0.17664, - "26": 0.17724, - "27": 0.17888, - "28": 0.17787, - "29": 0.17973, - "30": 0.17772, - "31": 0.17599, - "32": 0.17666, - "33": 0.17886, - "34": 0.17748, - "35": 0.17691, - "36": 0.17764, - "37": 0.17648, - "38": 0.17664, - "39": 0.17676, - "40": 0.17609, - "41": 0.17722, - "42": 0.1764, - "43": 0.1777, - "44": 0.17703, - "45": 0.17688, - "46": 0.17623, - "47": 0.17635, - "48": 0.17849, - "49": 0.17982, - "50": 0.17674, - "51": 0.22322, - "52": 0.2255, - "53": 0.18599, - "54": 0.18169, - "55": 0.18182, - "56": 0.18479, - "57": 0.1827, - "58": 0.18224, - "59": 0.1835, - "60": 0.18081, - "61": 0.18445, - "62": 0.18249, - "63": 0.18229, - "64": 0.18312, - "65": 0.18169, - "66": 0.18557, - "67": 0.18168, - "68": 0.18285, - "69": 0.18255, - "70": 0.18126, - "71": 0.18048, - "72": 0.18212, - "73": 0.18187, - "74": 0.18287, - "75": 0.18889, - "76": 0.1857, - "77": 0.1841, - "78": 0.18312, - "79": 0.18172, - "80": 0.18173, - "81": 0.18191, - "82": 0.18203, - "83": 0.18296, - "84": 0.18092, - "85": 0.1841, - "86": 0.1816, - "87": 0.18873, - "88": 0.1838, - "89": 0.18246, - "90": 0.18023, - "91": 0.18356, - "92": 0.18142, - "93": 0.18856, - "94": 0.18076, - "95": 0.17997, - "96": 0.18093, - "97": 0.18031, - "98": 0.18139, - "99": 0.18029, - "100": 0.17496 + "2": 5.11504, + "3": 0.19215, + "4": 0.18888, + "5": 0.18828, + "6": 0.18595, + "7": 0.18751, + "8": 0.18915, + "9": 0.18714, + "10": 0.18891, + "11": 0.18651, + "12": 0.18813, + "13": 0.18878, + "14": 0.18664, + "15": 0.18885, + "16": 0.18671, + "17": 0.18694, + "18": 0.1894, + "19": 0.18741, + "20": 0.18839, + "21": 0.18668, + "22": 0.19062, + "23": 0.18809, + "24": 0.18753, + "25": 0.18763, + "26": 0.18906, + "27": 0.18933, + "28": 0.19143, + "29": 0.19007, + "30": 0.18954, + "31": 0.18571, + "32": 0.18831, + "33": 0.19102, + "34": 0.19016, + "35": 0.18981, + "36": 0.18772, + "37": 0.18671, + "38": 0.18742, + "39": 0.18702, + "40": 0.1863, + "41": 0.23367, + "42": 0.18578, + "43": 0.18741, + "44": 0.18956, + "45": 0.18754, + "46": 0.19047, + "47": 0.20761, + "48": 0.18768, + "49": 0.19379, + "50": 0.18807, + "51": 0.28848, + "52": 0.23097, + "53": 0.19382, + "54": 0.19352, + "55": 0.1894, + "56": 0.19085, + "57": 0.18947, + "58": 0.19016, + "59": 0.19291, + "60": 0.19033, + "61": 0.1908, + "62": 0.19145, + "63": 0.19066, + "64": 0.18919, + "65": 0.1905, + "66": 0.19256, + "67": 0.19073, + "68": 0.1904, + "69": 0.19035, + "70": 0.18885, + "71": 0.1879, + "72": 0.18891, + "73": 0.1934, + "74": 0.18863, + "75": 0.18826, + "76": 0.19051, + "77": 0.18896, + "78": 0.18927, + "79": 0.18943, + "80": 0.18867, + "81": 0.18835, + "82": 0.18612, + "83": 0.18966, + "84": 0.20481, + "85": 0.20044, + "86": 0.20158, + "87": 0.19315, + "88": 0.19284, + "89": 0.19047, + "90": 0.19468, + "91": 0.1929, + "92": 0.19244, + "93": 0.19241, + "94": 0.1911, + "95": 0.19197, + "96": 0.19005, + "97": 0.1913, + "98": 0.19152, + "99": 0.19066, + "100": 0.19058 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json index 26b1b73478d..535fba25077 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92866, - "4": 10.9288, - "5": 10.93088, - "6": 10.92771, - "7": 10.92336, - "8": 10.92159, - "9": 10.92031, + "2": 10.92832, + "3": 10.92864, + "4": 10.92888, + "5": 10.93089, + "6": 10.92772, + "7": 10.92341, + "8": 10.92161, + "9": 10.92027, "10": 10.91964, - "11": 10.91027, - "12": 10.91814, - "13": 10.91523, - "14": 10.89825, - "15": 10.88042, - "16": 10.86841, + "11": 10.91024, + "12": 10.91826, + "13": 10.91526, + "14": 10.89832, + "15": 10.88051, + "16": 10.8684, "17": 10.86882, - "18": 10.86258, - "19": 10.86185, - "20": 10.77348, - "21": 10.75516, - "22": 10.75521, - "23": 10.74508, - "24": 10.71106, - "25": 10.71082, - "26": 10.69968, - "27": 10.65192, - "28": 10.58249, - "29": 10.55636, - "30": 10.54863, - "31": 10.52499, - "32": 10.51761, - "33": 10.48144, - "34": 10.451, - "35": 10.44994, - "36": 10.43486, - "37": 10.39411, - "38": 10.39546, - "39": 10.36016, - "40": 10.35798, - "41": 10.32782, - "42": 10.30704, - "43": 10.28206, - "44": 10.25604, - "45": 10.27039, - "46": 10.23404, - "47": 10.21882, + "18": 10.86249, + "19": 10.86187, + "20": 10.77339, + "21": 10.75518, + "22": 10.75531, + "23": 10.74509, + "24": 10.71112, + "25": 10.71086, + "26": 10.69962, + "27": 10.65199, + "28": 10.58247, + "29": 10.55645, + "30": 10.54865, + "31": 10.52506, + "32": 10.51769, + "33": 10.48149, + "34": 10.45111, + "35": 10.44997, + "36": 10.43493, + "37": 10.3941, + "38": 10.39549, + "39": 10.36015, + "40": 10.35806, + "41": 10.32785, + "42": 10.30705, + "43": 10.28217, + "44": 10.25607, + "45": 10.27045, + "46": 10.23413, + "47": 10.21893, "48": 10.17023, - "49": 10.1759, - "50": 10.17855, - "51": 10.19228, - "52": 10.13723, - "53": 10.13552, - "54": 10.10521, - "55": 10.08055, - "56": 10.11116, - "57": 10.09599, - "58": 10.10931, - "59": 10.05623, - "60": 10.07399, - "61": 10.02775, - "62": 10.0003, - "63": 10.07322, - "64": 10.03695, - "65": 10.00341, - "66": 10.03038, - "67": 10.0072, - "68": 9.97019, - "69": 9.9915, - "70": 9.97453, - "71": 10.00065, - "72": 9.97779, - "73": 9.97166, - "74": 9.95604, - "75": 9.93149, - "76": 9.96344, - "77": 9.95639, - "78": 9.90552, - "79": 9.91218, - "80": 9.92898, - "81": 9.95655, - "82": 9.89202, - "83": 9.85589, - "84": 9.78695, - "85": 9.78572, - "86": 9.88392, - "87": 9.91028, - "88": 9.88719, - "89": 9.81819, - "90": 9.81249, - "91": 9.82779, - "92": 9.81659, - "93": 9.74998, - "94": 9.8309, - "95": 9.82648, - "96": 9.80702, - "97": 9.74393, - "98": 9.77873, - "99": 9.82058, - "100": 9.71398 + "49": 10.17594, + "50": 10.17859, + "51": 10.19235, + "52": 10.13725, + "53": 10.13558, + "54": 10.10524, + "55": 10.08058, + "56": 10.11123, + "57": 10.09601, + "58": 10.10934, + "59": 10.05632, + "60": 10.07401, + "61": 10.02777, + "62": 10.00036, + "63": 10.07317, + "64": 10.03696, + "65": 10.00345, + "66": 10.0304, + "67": 10.00724, + "68": 9.97024, + "69": 9.99153, + "70": 9.9746, + "71": 10.00066, + "72": 9.97785, + "73": 9.9717, + "74": 9.95607, + "75": 9.93157, + "76": 9.96345, + "77": 9.95644, + "78": 9.90557, + "79": 9.9122, + "80": 9.92906, + "81": 9.95666, + "82": 9.89201, + "83": 9.85599, + "84": 9.78701, + "85": 9.7858, + "86": 9.88398, + "87": 9.91034, + "88": 9.88728, + "89": 9.81818, + "90": 9.81251, + "91": 9.82784, + "92": 9.81667, + "93": 9.75004, + "94": 9.83094, + "95": 9.82651, + "96": 9.80705, + "97": 9.74396, + "98": 9.77876, + "99": 9.82056, + "100": 9.71405 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1731.0, - "2": 1691.0, - "3": 1669.0, - "4": 1581.0, - "5": 1644.0, - "6": 1685.0, - "7": 1815.0, - "8": 1709.0, - "9": 1781.0, - "10": 1748.0, - "11": 1630.0, - "12": 1732.0, - "13": 1742.0, - "14": 1815.0, - "15": 1549.0, - "16": 1654.0, - "17": 1749.0, - "18": 1742.0, - "19": 1686.0, - "20": 1643.0, - "21": 1695.0, - "22": 1733.0, - "23": 1658.0, - "24": 1781.0, - "25": 1666.0, - "26": 1789.0, - "27": 1777.0, - "28": 1803.0, - "29": 1831.0, - "30": 1831.0, - "31": 2000.0, - "32": 1953.0, - "33": 1927.0, - "34": 2012.0, - "35": 2071.0, - "36": 1876.0, - "37": 2178.0, - "38": 2133.0, - "39": 2155.0, - "40": 2357.0, - "41": 2339.0, - "42": 1887.0, - "43": 2351.0, - "44": 2221.0, - "45": 2475.0, - "46": 2480.0, - "47": 2497.0, + "1": 1684.0, + "2": 1628.0, + "3": 1753.0, + "4": 1661.0, + "5": 1709.0, + "6": 1619.0, + "7": 1824.0, + "8": 1706.0, + "9": 1707.0, + "10": 1648.0, + "11": 1619.0, + "12": 1773.0, + "13": 1782.0, + "14": 1928.0, + "15": 1597.0, + "16": 1623.0, + "17": 1698.0, + "18": 1750.0, + "19": 1626.0, + "20": 1618.0, + "21": 1771.0, + "22": 1706.0, + "23": 1626.0, + "24": 1718.0, + "25": 1588.0, + "26": 1757.0, + "27": 1832.0, + "28": 1744.0, + "29": 1941.0, + "30": 1853.0, + "31": 1889.0, + "32": 1871.0, + "33": 1977.0, + "34": 2042.0, + "35": 2009.0, + "36": 1943.0, + "37": 2226.0, + "38": 2146.0, + "39": 2107.0, + "40": 2290.0, + "41": 2277.0, + "42": 1917.0, + "43": 2428.0, + "44": 2234.0, + "45": 2588.0, + "46": 2488.0, + "47": 2491.0, "48": 2601.0, - "49": 2909.0, - "50": 2638.0, - "51": 2495.0, - "52": 2790.0, - "53": 2729.0, - "54": 2777.0, - "55": 2626.0, - "56": 2743.0, - "57": 2177.0, - "58": 3682.0, - "59": 2899.0, - "60": 2899.0, - "61": 2826.0, - "62": 3369.0, - "63": 3377.0, - "64": 3714.0, - "65": 2750.0, - "66": 3201.0, - "67": 3778.0, - "68": 3542.0, - "69": 3015.0, - "70": 3264.0, - "71": 3129.0, - "72": 3054.0, - "73": 3365.0, - "74": 3234.0, - "75": 3345.0, - "76": 3367.0, - "77": 3958.0, - "78": 3329.0, - "79": 3194.0, - "80": 2951.0, - "81": 3592.0, - "82": 2943.0, - "83": 3073.0, - "84": 3037.0, - "85": 2541.0, - "86": 3023.0, - "87": 2916.0, - "88": 3027.0, - "89": 3180.0, - "90": 3778.0, - "91": 3014.0, - "92": 2726.0, - "93": 3051.0, - "94": 3000.0, - "95": 3189.0, - "96": 3382.0, - "97": 3490.0, - "98": 3305.0, - "99": 3298.0, - "100": 3311.0 + "49": 2860.0, + "50": 2622.0, + "51": 2566.0, + "52": 2713.0, + "53": 2694.0, + "54": 2837.0, + "55": 2660.0, + "56": 2691.0, + "57": 2100.0, + "58": 3764.0, + "59": 2897.0, + "60": 3048.0, + "61": 2813.0, + "62": 3200.0, + "63": 3426.0, + "64": 3603.0, + "65": 2607.0, + "66": 3090.0, + "67": 3762.0, + "68": 3476.0, + "69": 3189.0, + "70": 3310.0, + "71": 3107.0, + "72": 3040.0, + "73": 3459.0, + "74": 3262.0, + "75": 3167.0, + "76": 3312.0, + "77": 3909.0, + "78": 3290.0, + "79": 3200.0, + "80": 3133.0, + "81": 3597.0, + "82": 2947.0, + "83": 3172.0, + "84": 2917.0, + "85": 2518.0, + "86": 3140.0, + "87": 2832.0, + "88": 3076.0, + "89": 3067.0, + "90": 3800.0, + "91": 3056.0, + "92": 2804.0, + "93": 3105.0, + "94": 3146.0, + "95": 3205.0, + "96": 3332.0, + "97": 3391.0, + "98": 3361.0, + "99": 3318.0, + "100": 3245.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.02392, - "3": 2.50573, - "4": 2.4125, - "5": 2.18239, - "6": 2.29587, - "7": 1.85413, - "8": 2.68302, - "9": 2.677, - "10": 2.76099, - "11": 2.05385, - "12": 1.68068, - "13": 1.92045, - "14": 2.56065, - "15": 1.90139, - "16": 1.57226, - "17": 2.41383, - "18": 2.14416, - "19": 2.05606, - "20": 2.16357, - "21": 2.13152, - "22": 1.50502, - "23": 2.08983, - "24": 1.63963, - "25": 2.29431, - "26": 2.83751, - "27": 1.76414, - "28": 2.39332, - "29": 1.62607, - "30": 1.67187, - "31": 1.6553, - "32": 2.55786, - "33": 2.06527, - "34": 2.06572, - "35": 2.81203, - "36": 2.55928, - "37": 2.06661, - "38": 1.38414, - "39": 3.16205, - "40": 1.82841, - "41": 2.00472, - "42": 2.51756, - "43": 1.66235, - "44": 1.38274, - "45": 2.299, - "46": 1.55558, - "47": 2.08796, - "48": 1.98617, - "49": 1.72974, - "50": 2.01108, - "51": 1.56822, - "52": 2.26704, - "53": 2.26243, - "54": 2.24808, - "55": 1.79302, - "56": 2.73831, - "57": 0.92359, - "58": 2.06055, - "59": 1.75395, - "60": 2.24972, - "61": 3.25644, - "62": 3.72146, - "63": 2.11213, - "64": 2.34726, - "65": 2.21863, - "66": 2.96183, - "67": 1.90196, - "68": 1.55317, - "69": 1.46562, - "70": 1.99154, - "71": 2.26399, - "72": 1.9531, - "73": 1.92068, - "74": 1.80936, - "75": 1.6175, - "76": 1.75104, - "77": 1.5674, - "78": 2.05942, - "79": 2.19012, - "80": 2.1471, - "81": 2.4255, - "82": 2.30913, - "83": 2.09438, - "84": 1.89029, - "85": 1.79793, - "86": 2.03875, - "87": 2.23952, - "88": 2.10309, - "89": 2.395, - "90": 1.68837, - "91": 2.3603, - "92": 2.54642, - "93": 1.76393, - "94": 2.36051, - "95": 1.82959, - "96": 1.69184, - "97": 1.76942, - "98": 2.12513, - "99": 2.47915, - "100": 1.81358 + "2": 5.40968, + "3": 2.64561, + "4": 2.26454, + "5": 1.68729, + "6": 1.8202, + "7": 2.04859, + "8": 2.21028, + "9": 2.27243, + "10": 1.97343, + "11": 2.26077, + "12": 1.8489, + "13": 2.06052, + "14": 2.35668, + "15": 1.85032, + "16": 1.8782, + "17": 2.50524, + "18": 2.04894, + "19": 2.02238, + "20": 1.89591, + "21": 2.10042, + "22": 1.78132, + "23": 1.953, + "24": 1.52331, + "25": 2.21883, + "26": 1.97363, + "27": 1.88385, + "28": 1.77814, + "29": 1.68748, + "30": 1.98188, + "31": 1.58492, + "32": 2.78612, + "33": 1.62205, + "34": 1.55449, + "35": 2.037, + "36": 2.21818, + "37": 1.98218, + "38": 1.37224, + "39": 2.31227, + "40": 2.01265, + "41": 1.97017, + "42": 2.58355, + "43": 1.70866, + "44": 1.89129, + "45": 2.17059, + "46": 1.84456, + "47": 1.90234, + "48": 2.11718, + "49": 1.6443, + "50": 2.504, + "51": 1.17327, + "52": 2.18386, + "53": 2.05614, + "54": 2.18495, + "55": 2.00659, + "56": 3.09545, + "57": 1.85209, + "58": 2.42291, + "59": 1.89717, + "60": 1.88923, + "61": 2.22159, + "62": 2.44229, + "63": 1.84347, + "64": 2.16067, + "65": 1.90243, + "66": 2.36788, + "67": 2.09896, + "68": 1.56704, + "69": 2.02793, + "70": 1.94603, + "71": 2.40626, + "72": 1.86385, + "73": 1.95335, + "74": 1.8791, + "75": 1.80936, + "76": 2.03582, + "77": 1.81319, + "78": 1.86106, + "79": 2.19468, + "80": 1.96365, + "81": 2.18661, + "82": 2.22841, + "83": 2.15093, + "84": 1.68391, + "85": 1.8715, + "86": 1.88241, + "87": 2.24327, + "88": 2.27593, + "89": 2.38365, + "90": 2.1814, + "91": 2.27026, + "92": 2.60446, + "93": 2.07977, + "94": 2.41278, + "95": 2.45559, + "96": 2.73309, + "97": 1.90748, + "98": 2.08241, + "99": 3.01393, + "100": 1.94226 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json index 194b1d83846..172e80113de 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95174, - "2": 10.94869, + "1": 10.95177, + "2": 10.94867, "3": 10.94863, "4": 10.94972, "5": 10.94752, - "6": 10.94473, - "7": 10.94653, - "8": 10.94474, - "9": 10.94189, - "10": 10.94477, - "11": 10.94104, - "12": 10.93297, - "13": 10.92309, - "14": 10.92594, - "15": 10.8997, - "16": 10.89084, - "17": 10.89814, - "18": 10.88338, - "19": 10.88983, - "20": 10.79733, - "21": 10.79296, - "22": 10.77998, - "23": 10.76962, - "24": 10.74573, - "25": 10.73534, - "26": 10.72509, - "27": 10.67795, - "28": 10.62002, - "29": 10.59163, - "30": 10.56328, - "31": 10.56511, - "32": 10.54391, - "33": 10.50913, + "6": 10.94481, + "7": 10.94654, + "8": 10.94475, + "9": 10.94192, + "10": 10.94475, + "11": 10.9411, + "12": 10.93294, + "13": 10.92301, + "14": 10.92591, + "15": 10.89965, + "16": 10.8908, + "17": 10.89826, + "18": 10.88342, + "19": 10.88985, + "20": 10.7974, + "21": 10.79298, + "22": 10.78007, + "23": 10.76967, + "24": 10.74584, + "25": 10.7353, + "26": 10.72505, + "27": 10.67792, + "28": 10.62003, + "29": 10.59162, + "30": 10.56332, + "31": 10.56518, + "32": 10.54399, + "33": 10.50912, "34": 10.48321, - "35": 10.46997, - "36": 10.44684, - "37": 10.42648, - "38": 10.43042, - "39": 10.39272, - "40": 10.38039, - "41": 10.35134, - "42": 10.32751, - "43": 10.31124, - "44": 10.28344, - "45": 10.29601, - "46": 10.24782, - "47": 10.2438, - "48": 10.19392, - "49": 10.1967, - "50": 10.18921, - "51": 10.19743, - "52": 10.15225, - "53": 10.15772, - "54": 10.12075, - "55": 10.09671, - "56": 10.12112, - "57": 10.1117, - "58": 10.12098, - "59": 10.0624, - "60": 10.09235, - "61": 10.04175, - "62": 10.00957, - "63": 10.07948, - "64": 10.03237, - "65": 10.00321, - "66": 10.03952, - "67": 10.0196, - "68": 9.9857, - "69": 10.00194, + "35": 10.46989, + "36": 10.44693, + "37": 10.42657, + "38": 10.43049, + "39": 10.39276, + "40": 10.38042, + "41": 10.35136, + "42": 10.32754, + "43": 10.31128, + "44": 10.28351, + "45": 10.29607, + "46": 10.24787, + "47": 10.24381, + "48": 10.19395, + "49": 10.19677, + "50": 10.18925, + "51": 10.19742, + "52": 10.15222, + "53": 10.15777, + "54": 10.12078, + "55": 10.09677, + "56": 10.12113, + "57": 10.11179, + "58": 10.12099, + "59": 10.06245, + "60": 10.09238, + "61": 10.04181, + "62": 10.0096, + "63": 10.07945, + "64": 10.03238, + "65": 10.00325, + "66": 10.03958, + "67": 10.01965, + "68": 9.98575, + "69": 10.00201, "70": 9.98145, - "71": 10.00455, - "72": 9.99623, - "73": 9.98762, - "74": 9.97285, - "75": 9.93231, + "71": 10.00458, + "72": 9.99624, + "73": 9.98765, + "74": 9.97292, + "75": 9.93235, "76": 9.96721, - "77": 9.96947, - "78": 9.92081, - "79": 9.91732, - "80": 9.93739, - "81": 9.95719, - "82": 9.89591, - "83": 9.86217, - "84": 9.80484, - "85": 9.78635, - "86": 9.89555, - "87": 9.90871, - "88": 9.88855, - "89": 9.83329, + "77": 9.96958, + "78": 9.92088, + "79": 9.91738, + "80": 9.93744, + "81": 9.95727, + "82": 9.89594, + "83": 9.86225, + "84": 9.80486, + "85": 9.78643, + "86": 9.89557, + "87": 9.90878, + "88": 9.88857, + "89": 9.83332, "90": 9.82425, "91": 9.83931, - "92": 9.82243, - "93": 9.75871, - "94": 9.83497, - "95": 9.82145, - "96": 9.81159, - "97": 9.75627, - "98": 9.78245, - "99": 9.82338, - "100": 9.71587 + "92": 9.82248, + "93": 9.75876, + "94": 9.83501, + "95": 9.82148, + "96": 9.8116, + "97": 9.75626, + "98": 9.78247, + "99": 9.82337, + "100": 9.71588 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 74.0, - "2": 75.0, - "3": 64.0, - "4": 85.0, - "5": 53.0, - "6": 73.0, - "7": 70.0, - "8": 69.0, - "9": 56.0, - "10": 54.0, - "11": 77.0, - "12": 82.0, - "13": 64.0, - "14": 71.0, - "15": 73.0, - "16": 68.0, - "17": 60.0, - "18": 85.0, - "19": 77.0, - "20": 66.0, - "21": 51.0, - "22": 65.0, - "23": 57.0, - "24": 75.0, - "25": 64.0, - "26": 75.0, - "27": 61.0, + "1": 52.0, + "2": 54.0, + "3": 67.0, + "4": 68.0, + "5": 50.0, + "6": 69.0, + "7": 69.0, + "8": 70.0, + "9": 68.0, + "10": 71.0, + "11": 78.0, + "12": 60.0, + "13": 79.0, + "14": 61.0, + "15": 71.0, + "16": 57.0, + "17": 52.0, + "18": 73.0, + "19": 57.0, + "20": 65.0, + "21": 82.0, + "22": 73.0, + "23": 73.0, + "24": 71.0, + "25": 51.0, + "26": 71.0, + "27": 57.0, "28": 64.0, - "29": 71.0, - "30": 58.0, - "31": 75.0, - "32": 58.0, - "33": 67.0, - "34": 74.0, - "35": 70.0, - "36": 85.0, - "37": 69.0, - "38": 77.0, - "39": 69.0, - "40": 87.0, - "41": 77.0, - "42": 79.0, - "43": 74.0, - "44": 95.0, - "45": 67.0, - "46": 63.0, - "47": 71.0, - "48": 76.0, - "49": 73.0, + "29": 70.0, + "30": 75.0, + "31": 89.0, + "32": 70.0, + "33": 69.0, + "34": 78.0, + "35": 79.0, + "36": 69.0, + "37": 83.0, + "38": 67.0, + "39": 71.0, + "40": 50.0, + "41": 79.0, + "42": 70.0, + "43": 83.0, + "44": 73.0, + "45": 87.0, + "46": 78.0, + "47": 90.0, + "48": 74.0, + "49": 84.0, "50": 79.0, - "51": 95.0, - "52": 84.0, - "53": 70.0, - "54": 91.0, - "55": 74.0, - "56": 78.0, - "57": 72.0, - "58": 78.0, - "59": 72.0, - "60": 86.0, - "61": 68.0, - "62": 98.0, - "63": 96.0, - "64": 93.0, - "65": 82.0, - "66": 66.0, - "67": 110.0, + "51": 102.0, + "52": 88.0, + "53": 89.0, + "54": 90.0, + "55": 75.0, + "56": 80.0, + "57": 60.0, + "58": 79.0, + "59": 66.0, + "60": 77.0, + "61": 83.0, + "62": 109.0, + "63": 72.0, + "64": 114.0, + "65": 86.0, + "66": 78.0, + "67": 102.0, "68": 96.0, - "69": 90.0, - "70": 101.0, - "71": 82.0, + "69": 79.0, + "70": 82.0, + "71": 68.0, "72": 76.0, - "73": 76.0, - "74": 87.0, - "75": 74.0, - "76": 74.0, - "77": 62.0, - "78": 77.0, - "79": 89.0, - "80": 68.0, - "81": 91.0, - "82": 73.0, - "83": 94.0, - "84": 90.0, - "85": 75.0, - "86": 94.0, - "87": 76.0, - "88": 80.0, - "89": 95.0, - "90": 96.0, - "91": 65.0, - "92": 65.0, - "93": 57.0, - "94": 78.0, - "95": 89.0, - "96": 60.0, - "97": 109.0, - "98": 56.0, - "99": 68.0, - "100": 99.0 + "73": 94.0, + "74": 75.0, + "75": 76.0, + "76": 82.0, + "77": 88.0, + "78": 119.0, + "79": 107.0, + "80": 82.0, + "81": 73.0, + "82": 72.0, + "83": 82.0, + "84": 80.0, + "85": 77.0, + "86": 77.0, + "87": 80.0, + "88": 77.0, + "89": 90.0, + "90": 92.0, + "91": 77.0, + "92": 72.0, + "93": 82.0, + "94": 72.0, + "95": 86.0, + "96": 83.0, + "97": 83.0, + "98": 74.0, + "99": 70.0, + "100": 72.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.05646, - "3": 1.64926, - "4": 1.51364, - "5": 0.87448, - "6": 0.8958, - "7": 1.13314, - "8": 1.69323, - "9": 1.97024, - "10": 0.889, - "11": 1.14997, - "12": 1.33119, - "13": 1.43251, - "14": 0.87966, - "15": 1.21883, - "16": 1.18965, - "17": 1.28078, - "18": 1.42888, - "19": 1.3082, - "20": 1.03096, - "21": 1.47204, - "22": 1.171, - "23": 1.21307, - "24": 1.75377, - "25": 0.91489, - "26": 1.11231, - "27": 1.4801, - "28": 1.77159, - "29": 0.80668, - "30": 1.10193, - "31": 0.97633, - "32": 1.61749, - "33": 1.55435, - "34": 1.15377, - "35": 1.71313, - "36": 0.91351, - "37": 1.14328, - "38": 1.37552, - "39": 1.5564, - "40": 2.68163, - "41": 1.41261, - "42": 1.38325, - "43": 1.2548, - "44": 2.14185, - "45": 1.32246, - "46": 1.27186, - "47": 1.57862, - "48": 0.87885, - "49": 1.5616, - "50": 1.51913, - "51": 0.20474, - "52": 1.49753, - "53": 1.41508, - "54": 1.50829, - "55": 1.53237, - "56": 1.13149, - "57": 1.31628, - "58": 1.41131, - "59": 1.15957, - "60": 1.23452, - "61": 1.61395, - "62": 1.09671, - "63": 1.39238, - "64": 1.398, - "65": 0.93338, - "66": 1.17371, - "67": 1.94759, - "68": 1.02385, - "69": 0.98696, - "70": 1.50737, - "71": 1.85133, - "72": 0.85571, - "73": 1.68757, - "74": 0.72292, - "75": 0.99087, - "76": 1.55542, - "77": 1.44187, - "78": 0.88968, - "79": 2.01173, - "80": 1.55881, - "81": 1.43639, - "82": 0.77707, - "83": 1.28231, - "84": 1.32696, - "85": 0.98777, - "86": 1.41843, - "87": 1.71853, - "88": 1.38841, - "89": 1.61498, - "90": 1.6885, - "91": 1.05804, - "92": 1.55219, - "93": 1.57285, - "94": 1.46847, - "95": 1.25271, - "96": 1.32851, - "97": 1.26383, - "98": 1.35329, - "99": 2.01532, - "100": 1.47822 + "2": 5.18237, + "3": 0.19713, + "4": 0.19611, + "5": 0.19801, + "6": 0.19477, + "7": 0.19247, + "8": 0.19472, + "9": 0.19364, + "10": 0.19416, + "11": 0.20537, + "12": 0.20245, + "13": 0.19969, + "14": 0.19466, + "15": 0.19458, + "16": 0.19671, + "17": 0.19541, + "18": 0.196, + "19": 0.20121, + "20": 0.19664, + "21": 0.19775, + "22": 0.19765, + "23": 0.19613, + "24": 0.19745, + "25": 0.19548, + "26": 0.19747, + "27": 0.19578, + "28": 0.19659, + "29": 0.19586, + "30": 0.19881, + "31": 0.19932, + "32": 0.19407, + "33": 0.20014, + "34": 0.19555, + "35": 0.1954, + "36": 0.19539, + "37": 0.19582, + "38": 0.195, + "39": 0.19577, + "40": 0.1952, + "41": 0.19701, + "42": 0.19528, + "43": 0.20028, + "44": 0.19531, + "45": 0.19467, + "46": 0.19552, + "47": 0.19578, + "48": 0.19833, + "49": 0.20115, + "50": 0.19534, + "51": 0.22676, + "52": 0.23261, + "53": 0.19561, + "54": 0.19497, + "55": 0.193, + "56": 0.19531, + "57": 0.19199, + "58": 0.19263, + "59": 0.19434, + "60": 0.1949, + "61": 0.19407, + "62": 0.19598, + "63": 0.19291, + "64": 0.1935, + "65": 0.19185, + "66": 0.19368, + "67": 0.19063, + "68": 0.19446, + "69": 0.19498, + "70": 0.1918, + "71": 0.19216, + "72": 0.19116, + "73": 0.19647, + "74": 0.19425, + "75": 0.19195, + "76": 0.19741, + "77": 0.19207, + "78": 0.19344, + "79": 0.19436, + "80": 0.1927, + "81": 0.19249, + "82": 0.19279, + "83": 0.19387, + "84": 0.19441, + "85": 0.19519, + "86": 0.19088, + "87": 0.19384, + "88": 0.19465, + "89": 0.1931, + "90": 0.19166, + "91": 0.19346, + "92": 0.19183, + "93": 0.19309, + "94": 0.18989, + "95": 0.18974, + "96": 0.19066, + "97": 0.19183, + "98": 0.19138, + "99": 0.19061, + "100": 0.19011 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json index 06a2c6615e5..9e90f74cdf0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute_1node/golden_values_dev_dgx_gb200.json @@ -5,105 +5,105 @@ "step_interval": 1, "values": { "1": 10.93086, - "2": 10.92833, - "3": 10.92867, - "4": 10.92885, - "5": 10.93089, - "6": 10.92768, - "7": 10.92339, - "8": 10.92161, - "9": 10.92031, - "10": 10.91963, - "11": 10.91026, - "12": 10.91814, - "13": 10.91528, - "14": 10.89825, - "15": 10.88044, - "16": 10.86845, - "17": 10.86882, - "18": 10.86255, - "19": 10.86184, - "20": 10.77349, - "21": 10.75515, - "22": 10.75525, - "23": 10.7451, - "24": 10.71105, - "25": 10.71078, - "26": 10.69967, - "27": 10.65193, - "28": 10.5825, - "29": 10.55638, - "30": 10.54863, - "31": 10.52498, - "32": 10.51762, - "33": 10.48146, - "34": 10.45103, - "35": 10.44996, - "36": 10.43489, - "37": 10.39411, - "38": 10.39543, - "39": 10.36014, - "40": 10.35802, - "41": 10.32785, - "42": 10.307, - "43": 10.28209, - "44": 10.25605, - "45": 10.27043, - "46": 10.23405, - "47": 10.21883, - "48": 10.17023, - "49": 10.17586, - "50": 10.17856, - "51": 10.19226, - "52": 10.13722, - "53": 10.13551, - "54": 10.10522, - "55": 10.08051, - "56": 10.11117, - "57": 10.09597, - "58": 10.10933, - "59": 10.05625, - "60": 10.07399, - "61": 10.02772, - "62": 10.00029, - "63": 10.07319, - "64": 10.0369, - "65": 10.00343, + "2": 10.92832, + "3": 10.92865, + "4": 10.92887, + "5": 10.93092, + "6": 10.92771, + "7": 10.9234, + "8": 10.92162, + "9": 10.9203, + "10": 10.91967, + "11": 10.91022, + "12": 10.91827, + "13": 10.91529, + "14": 10.8983, + "15": 10.88051, + "16": 10.86839, + "17": 10.86885, + "18": 10.86249, + "19": 10.86187, + "20": 10.77337, + "21": 10.75517, + "22": 10.75534, + "23": 10.74513, + "24": 10.71113, + "25": 10.71087, + "26": 10.69964, + "27": 10.65196, + "28": 10.58251, + "29": 10.55639, + "30": 10.54867, + "31": 10.52503, + "32": 10.51767, + "33": 10.48145, + "34": 10.45107, + "35": 10.44997, + "36": 10.43491, + "37": 10.39413, + "38": 10.39554, + "39": 10.36016, + "40": 10.35805, + "41": 10.32787, + "42": 10.30705, + "43": 10.28218, + "44": 10.25608, + "45": 10.27046, + "46": 10.23411, + "47": 10.21893, + "48": 10.17027, + "49": 10.17593, + "50": 10.17859, + "51": 10.19236, + "52": 10.13724, + "53": 10.13555, + "54": 10.10525, + "55": 10.08055, + "56": 10.11124, + "57": 10.096, + "58": 10.10935, + "59": 10.05629, + "60": 10.07402, + "61": 10.02777, + "62": 10.00036, + "63": 10.07322, + "64": 10.03699, + "65": 10.00348, "66": 10.03036, "67": 10.00721, - "68": 9.97016, + "68": 9.97027, "69": 9.99154, - "70": 9.97454, - "71": 10.00064, - "72": 9.97777, - "73": 9.97168, - "74": 9.956, - "75": 9.93149, - "76": 9.96344, - "77": 9.95634, - "78": 9.90551, - "79": 9.91217, - "80": 9.92898, - "81": 9.95658, - "82": 9.89201, - "83": 9.85591, - "84": 9.78695, - "85": 9.78569, - "86": 9.88392, - "87": 9.91031, - "88": 9.88719, - "89": 9.81821, - "90": 9.81249, + "70": 9.97455, + "71": 10.00068, + "72": 9.97786, + "73": 9.97171, + "74": 9.95605, + "75": 9.93157, + "76": 9.96343, + "77": 9.9564, + "78": 9.90555, + "79": 9.91225, + "80": 9.92908, + "81": 9.95664, + "82": 9.892, + "83": 9.85597, + "84": 9.787, + "85": 9.78578, + "86": 9.88395, + "87": 9.91032, + "88": 9.88725, + "89": 9.81819, + "90": 9.8125, "91": 9.82781, - "92": 9.81661, - "93": 9.74996, - "94": 9.83091, - "95": 9.82648, - "96": 9.80705, - "97": 9.7439, - "98": 9.77871, - "99": 9.82058, - "100": 9.71401 + "92": 9.81663, + "93": 9.75004, + "94": 9.83095, + "95": 9.82651, + "96": 9.80706, + "97": 9.74396, + "98": 9.77875, + "99": 9.82057, + "100": 9.71407 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 76.0, - "2": 63.0, - "3": 65.0, - "4": 73.0, - "5": 70.0, - "6": 60.0, - "7": 65.0, - "8": 61.0, - "9": 60.0, - "10": 68.0, - "11": 69.0, - "12": 55.0, - "13": 72.0, - "14": 74.0, - "15": 52.0, - "16": 66.0, - "17": 56.0, - "18": 62.0, - "19": 57.0, + "1": 59.0, + "2": 45.0, + "3": 78.0, + "4": 62.0, + "5": 64.0, + "6": 58.0, + "7": 62.0, + "8": 54.0, + "9": 69.0, + "10": 67.0, + "11": 62.0, + "12": 70.0, + "13": 64.0, + "14": 50.0, + "15": 58.0, + "16": 61.0, + "17": 70.0, + "18": 52.0, + "19": 58.0, "20": 72.0, - "21": 60.0, - "22": 60.0, - "23": 57.0, - "24": 72.0, - "25": 71.0, - "26": 77.0, + "21": 57.0, + "22": 63.0, + "23": 70.0, + "24": 75.0, + "25": 59.0, + "26": 58.0, "27": 66.0, - "28": 68.0, - "29": 63.0, - "30": 68.0, - "31": 80.0, + "28": 55.0, + "29": 57.0, + "30": 85.0, + "31": 76.0, "32": 75.0, - "33": 60.0, - "34": 71.0, - "35": 74.0, - "36": 76.0, - "37": 70.0, - "38": 64.0, - "39": 76.0, - "40": 78.0, - "41": 72.0, - "42": 81.0, - "43": 73.0, - "44": 87.0, - "45": 94.0, - "46": 59.0, - "47": 76.0, - "48": 82.0, - "49": 75.0, - "50": 80.0, - "51": 74.0, - "52": 76.0, - "53": 76.0, - "54": 68.0, - "55": 58.0, - "56": 68.0, - "57": 73.0, - "58": 104.0, - "59": 94.0, - "60": 84.0, - "61": 85.0, - "62": 77.0, - "63": 71.0, - "64": 92.0, - "65": 78.0, - "66": 97.0, - "67": 89.0, - "68": 82.0, - "69": 98.0, - "70": 82.0, - "71": 96.0, - "72": 87.0, + "33": 85.0, + "34": 78.0, + "35": 81.0, + "36": 65.0, + "37": 64.0, + "38": 76.0, + "39": 83.0, + "40": 72.0, + "41": 88.0, + "42": 60.0, + "43": 82.0, + "44": 78.0, + "45": 69.0, + "46": 74.0, + "47": 72.0, + "48": 87.0, + "49": 83.0, + "50": 83.0, + "51": 87.0, + "52": 88.0, + "53": 88.0, + "54": 78.0, + "55": 90.0, + "56": 64.0, + "57": 79.0, + "58": 96.0, + "59": 89.0, + "60": 69.0, + "61": 96.0, + "62": 68.0, + "63": 79.0, + "64": 94.0, + "65": 86.0, + "66": 78.0, + "67": 86.0, + "68": 86.0, + "69": 76.0, + "70": 80.0, + "71": 83.0, + "72": 82.0, "73": 94.0, - "74": 80.0, - "75": 80.0, - "76": 69.0, - "77": 100.0, - "78": 88.0, - "79": 90.0, - "80": 70.0, - "81": 102.0, - "82": 60.0, - "83": 93.0, - "84": 85.0, - "85": 69.0, - "86": 81.0, - "87": 75.0, - "88": 80.0, - "89": 84.0, - "90": 81.0, - "91": 74.0, - "92": 62.0, - "93": 65.0, - "94": 81.0, - "95": 91.0, - "96": 83.0, - "97": 88.0, - "98": 66.0, - "99": 72.0, - "100": 91.0 + "74": 83.0, + "75": 70.0, + "76": 65.0, + "77": 90.0, + "78": 85.0, + "79": 94.0, + "80": 81.0, + "81": 98.0, + "82": 52.0, + "83": 98.0, + "84": 80.0, + "85": 86.0, + "86": 77.0, + "87": 73.0, + "88": 68.0, + "89": 88.0, + "90": 86.0, + "91": 82.0, + "92": 56.0, + "93": 80.0, + "94": 72.0, + "95": 83.0, + "96": 75.0, + "97": 84.0, + "98": 77.0, + "99": 98.0, + "100": 77.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.03088, - "3": 3.34876, - "4": 2.53582, - "5": 2.48058, - "6": 2.43762, - "7": 1.87523, - "8": 2.50536, - "9": 2.65386, - "10": 2.04284, - "11": 2.73294, - "12": 1.82067, - "13": 2.48704, - "14": 2.68492, - "15": 1.98056, - "16": 2.2973, - "17": 2.95035, - "18": 2.40728, - "19": 2.51638, - "20": 2.29895, - "21": 2.79292, - "22": 2.5803, - "23": 3.3125, - "24": 2.54299, - "25": 2.96965, - "26": 3.57266, - "27": 2.68069, - "28": 2.62391, - "29": 2.78523, - "30": 2.0347, - "31": 1.80005, - "32": 3.21385, - "33": 1.71827, - "34": 1.87217, - "35": 2.78285, - "36": 2.60662, - "37": 2.08884, - "38": 1.57836, - "39": 2.23851, - "40": 2.02535, - "41": 1.9462, - "42": 2.95662, - "43": 2.26434, - "44": 2.28424, - "45": 2.64872, - "46": 1.94126, - "47": 2.08261, - "48": 2.90839, - "49": 2.16688, - "50": 2.52428, - "51": 2.10448, - "52": 2.34448, - "53": 2.09852, - "54": 2.50284, - "55": 1.98433, - "56": 3.40748, - "57": 1.51938, - "58": 2.28296, - "59": 1.91628, - "60": 1.80878, - "61": 2.36241, - "62": 2.24451, - "63": 1.64096, - "64": 2.12785, - "65": 2.06667, - "66": 2.38939, - "67": 2.48401, - "68": 1.73357, - "69": 2.38218, - "70": 2.32129, - "71": 3.13742, - "72": 2.27983, - "73": 2.21754, - "74": 2.47275, - "75": 2.27557, - "76": 3.45149, - "77": 3.03361, - "78": 2.35039, - "79": 3.48119, - "80": 2.39514, - "81": 2.94682, - "82": 2.42092, - "83": 2.52566, - "84": 2.19555, - "85": 2.57184, - "86": 2.11493, - "87": 2.91445, - "88": 2.56316, - "89": 2.76157, - "90": 2.28355, - "91": 2.23103, - "92": 2.74864, - "93": 2.04853, - "94": 2.46703, - "95": 2.82977, - "96": 2.48175, - "97": 2.19588, - "98": 2.00735, - "99": 3.01988, - "100": 2.2927 + "2": 5.07731, + "3": 1.92395, + "4": 1.9626, + "5": 1.51896, + "6": 1.56971, + "7": 2.00612, + "8": 2.32895, + "9": 2.42806, + "10": 2.06118, + "11": 2.00439, + "12": 1.66027, + "13": 1.90066, + "14": 1.94035, + "15": 1.57455, + "16": 1.95098, + "17": 2.2236, + "18": 1.89713, + "19": 1.85551, + "20": 1.47328, + "21": 2.05525, + "22": 1.67582, + "23": 1.83112, + "24": 1.52929, + "25": 2.09567, + "26": 2.27405, + "27": 1.57156, + "28": 1.82613, + "29": 1.56668, + "30": 1.89443, + "31": 1.82977, + "32": 2.91141, + "33": 1.69788, + "34": 1.43212, + "35": 1.90688, + "36": 2.01091, + "37": 1.71473, + "38": 1.55316, + "39": 2.36082, + "40": 2.14634, + "41": 2.12827, + "42": 2.49928, + "43": 1.91366, + "44": 1.79483, + "45": 2.04944, + "46": 1.63766, + "47": 1.82571, + "48": 2.19023, + "49": 1.86445, + "50": 2.53233, + "51": 1.1974, + "52": 2.00422, + "53": 1.86547, + "54": 2.10392, + "55": 1.8704, + "56": 2.09548, + "57": 1.2313, + "58": 2.06096, + "59": 1.53726, + "60": 1.46344, + "61": 2.20518, + "62": 2.2589, + "63": 1.69762, + "64": 1.59037, + "65": 1.85351, + "66": 2.34147, + "67": 2.2177, + "68": 1.62013, + "69": 2.1971, + "70": 2.18011, + "71": 2.59915, + "72": 1.99488, + "73": 1.9122, + "74": 1.98433, + "75": 1.56168, + "76": 1.82549, + "77": 1.92207, + "78": 1.84572, + "79": 2.20454, + "80": 1.9514, + "81": 2.41403, + "82": 2.07208, + "83": 2.15301, + "84": 1.55133, + "85": 1.94879, + "86": 1.83192, + "87": 2.28935, + "88": 1.99803, + "89": 2.13387, + "90": 1.97378, + "91": 1.9011, + "92": 2.27493, + "93": 1.61459, + "94": 1.87748, + "95": 1.76718, + "96": 1.81554, + "97": 1.56225, + "98": 2.19203, + "99": 2.23269, + "100": 1.91488 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json index b111a92e440..146d89ba004 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.9313, - "2": 10.92649, - "3": 10.92492, - "4": 10.92049, - "5": 10.91862, - "6": 10.91576, - "7": 10.9247, - "8": 10.91818, - "9": 10.9151, - "10": 10.91994, - "11": 10.91158, - "12": 10.90479, - "13": 10.90054, - "14": 10.9013, - "15": 10.88359, + "1": 10.93124, + "2": 10.92656, + "3": 10.92483, + "4": 10.92047, + "5": 10.9186, + "6": 10.91565, + "7": 10.92468, + "8": 10.91821, + "9": 10.91516, + "10": 10.91989, + "11": 10.91162, + "12": 10.90475, + "13": 10.90051, + "14": 10.90133, + "15": 10.88358, "16": 10.8637, - "17": 10.88307, - "18": 10.86572, - "19": 10.86865, - "20": 10.79164, - "21": 10.7875, - "22": 10.7756, - "23": 10.76381, - "24": 10.73309, - "25": 10.73329, - "26": 10.71734, - "27": 10.6737, - "28": 10.61526, - "29": 10.58911, - "30": 10.5584, - "31": 10.56339, + "17": 10.88315, + "18": 10.86578, + "19": 10.86864, + "20": 10.79161, + "21": 10.78749, + "22": 10.77557, + "23": 10.76385, + "24": 10.73307, + "25": 10.73334, + "26": 10.71736, + "27": 10.67378, + "28": 10.61523, + "29": 10.5891, + "30": 10.55845, + "31": 10.56337, "32": 10.54733, - "33": 10.51065, - "34": 10.48411, - "35": 10.48015, - "36": 10.45812, - "37": 10.43332, - "38": 10.43472, - "39": 10.39235, - "40": 10.38663, - "41": 10.36414, + "33": 10.51066, + "34": 10.48413, + "35": 10.48022, + "36": 10.45815, + "37": 10.43336, + "38": 10.43487, + "39": 10.39242, + "40": 10.38667, + "41": 10.36427, "42": 10.34637, - "43": 10.32167, - "44": 10.29975, - "45": 10.30678, - "46": 10.27001, - "47": 10.25529, - "48": 10.20947, - "49": 10.21406, - "50": 10.20718, - "51": 10.20963, - "52": 10.16215, + "43": 10.32175, + "44": 10.2997, + "45": 10.30682, + "46": 10.27007, + "47": 10.25537, + "48": 10.20952, + "49": 10.21412, + "50": 10.20724, + "51": 10.20967, + "52": 10.1622, "53": 10.16867, - "54": 10.13116, - "55": 10.10553, - "56": 10.13037, - "57": 10.1174, - "58": 10.12717, - "59": 10.07187, - "60": 10.09529, + "54": 10.13123, + "55": 10.10561, + "56": 10.1304, + "57": 10.11749, + "58": 10.12723, + "59": 10.07194, + "60": 10.09526, "61": 10.04934, - "62": 10.01408, - "63": 10.08517, - "64": 10.0346, - "65": 10.00773, - "66": 10.0456, - "67": 10.02072, - "68": 9.98561, - "69": 9.99975, - "70": 9.98442, + "62": 10.01411, + "63": 10.08524, + "64": 10.03467, + "65": 10.00779, + "66": 10.04566, + "67": 10.02073, + "68": 9.98565, + "69": 9.99973, + "70": 9.98439, "71": 10.00594, - "72": 9.99176, - "73": 9.98241, - "74": 9.9695, - "75": 9.92723, + "72": 9.9918, + "73": 9.98247, + "74": 9.96949, + "75": 9.92731, "76": 9.96628, - "77": 9.97061, - "78": 9.91891, - "79": 9.91537, - "80": 9.93334, - "81": 9.95468, - "82": 9.89573, - "83": 9.86367, - "84": 9.80477, - "85": 9.78775, + "77": 9.97068, + "78": 9.91894, + "79": 9.91539, + "80": 9.93338, + "81": 9.95475, + "82": 9.89574, + "83": 9.8637, + "84": 9.80473, + "85": 9.7878, "86": 9.89713, - "87": 9.90766, - "88": 9.88965, - "89": 9.83169, + "87": 9.90773, + "88": 9.88967, + "89": 9.83167, "90": 9.82537, - "91": 9.83475, - "92": 9.8245, - "93": 9.75847, - "94": 9.83101, - "95": 9.81948, - "96": 9.80777, - "97": 9.75345, - "98": 9.77888, - "99": 9.82077, - "100": 9.71277 + "91": 9.83482, + "92": 9.82443, + "93": 9.75854, + "94": 9.83105, + "95": 9.81946, + "96": 9.80783, + "97": 9.75353, + "98": 9.77882, + "99": 9.82076, + "100": 9.71284 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1952.0, - "2": 1817.0, - "3": 1808.0, - "4": 1791.0, - "5": 1885.0, + "1": 1979.0, + "2": 1748.0, + "3": 1815.0, + "4": 1776.0, + "5": 1902.0, "6": 1821.0, - "7": 2063.0, - "8": 1849.0, - "9": 1891.0, - "10": 1826.0, - "11": 1829.0, - "12": 1803.0, - "13": 1971.0, - "14": 1928.0, - "15": 1702.0, - "16": 1851.0, - "17": 1860.0, - "18": 1860.0, - "19": 1847.0, - "20": 1759.0, - "21": 2068.0, - "22": 1814.0, - "23": 1824.0, - "24": 1928.0, - "25": 1823.0, - "26": 1823.0, - "27": 1934.0, - "28": 2025.0, - "29": 2099.0, - "30": 1913.0, - "31": 2038.0, - "32": 2141.0, - "33": 2100.0, - "34": 2173.0, - "35": 2211.0, - "36": 2318.0, - "37": 2437.0, - "38": 2196.0, - "39": 2330.0, - "40": 2327.0, - "41": 2312.0, - "42": 2141.0, - "43": 2494.0, - "44": 2427.0, - "45": 2658.0, - "46": 2486.0, - "47": 2586.0, - "48": 2662.0, - "49": 2805.0, - "50": 2596.0, - "51": 2668.0, - "52": 2908.0, - "53": 2663.0, - "54": 3035.0, - "55": 2510.0, - "56": 2886.0, - "57": 2353.0, - "58": 3662.0, - "59": 2903.0, - "60": 3034.0, - "61": 2731.0, - "62": 3185.0, - "63": 3430.0, - "64": 3798.0, - "65": 2668.0, - "66": 2992.0, - "67": 3917.0, - "68": 3432.0, - "69": 3100.0, - "70": 3370.0, - "71": 3312.0, - "72": 3110.0, - "73": 3537.0, - "74": 3315.0, - "75": 3231.0, - "76": 3512.0, - "77": 3745.0, - "78": 3437.0, - "79": 3539.0, - "80": 3091.0, - "81": 3446.0, - "82": 3428.0, - "83": 3132.0, - "84": 3071.0, - "85": 2856.0, - "86": 3102.0, - "87": 3132.0, - "88": 3289.0, - "89": 3119.0, - "90": 4321.0, - "91": 3021.0, - "92": 3215.0, - "93": 3117.0, - "94": 3434.0, - "95": 3353.0, - "96": 3968.0, - "97": 3613.0, - "98": 3539.0, - "99": 3548.0, - "100": 3423.0 + "7": 2096.0, + "8": 1814.0, + "9": 1986.0, + "10": 1854.0, + "11": 1841.0, + "12": 1830.0, + "13": 1959.0, + "14": 1959.0, + "15": 1732.0, + "16": 1757.0, + "17": 1857.0, + "18": 1843.0, + "19": 1871.0, + "20": 1811.0, + "21": 1908.0, + "22": 1769.0, + "23": 1916.0, + "24": 1982.0, + "25": 1724.0, + "26": 1815.0, + "27": 1991.0, + "28": 2083.0, + "29": 2080.0, + "30": 1957.0, + "31": 2135.0, + "32": 2036.0, + "33": 2046.0, + "34": 2092.0, + "35": 2270.0, + "36": 2223.0, + "37": 2385.0, + "38": 2130.0, + "39": 2311.0, + "40": 2405.0, + "41": 2399.0, + "42": 2119.0, + "43": 2534.0, + "44": 2262.0, + "45": 2600.0, + "46": 2520.0, + "47": 2549.0, + "48": 2614.0, + "49": 2830.0, + "50": 2624.0, + "51": 2573.0, + "52": 2725.0, + "53": 2640.0, + "54": 2861.0, + "55": 2646.0, + "56": 2801.0, + "57": 2420.0, + "58": 3678.0, + "59": 2935.0, + "60": 3002.0, + "61": 2701.0, + "62": 3221.0, + "63": 3238.0, + "64": 3778.0, + "65": 2746.0, + "66": 3130.0, + "67": 3794.0, + "68": 3423.0, + "69": 3104.0, + "70": 3405.0, + "71": 3326.0, + "72": 3135.0, + "73": 3547.0, + "74": 3247.0, + "75": 3221.0, + "76": 3389.0, + "77": 3780.0, + "78": 3334.0, + "79": 3528.0, + "80": 3092.0, + "81": 3390.0, + "82": 3497.0, + "83": 3199.0, + "84": 3098.0, + "85": 2849.0, + "86": 3093.0, + "87": 3255.0, + "88": 3214.0, + "89": 3222.0, + "90": 4190.0, + "91": 2958.0, + "92": 3147.0, + "93": 3180.0, + "94": 3453.0, + "95": 3333.0, + "96": 3907.0, + "97": 3640.0, + "98": 3579.0, + "99": 3534.0, + "100": 3385.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 2.98595, - "3": 0.22249, - "4": 0.21742, - "5": 0.22431, - "6": 0.21965, - "7": 0.22012, - "8": 0.21964, - "9": 0.21811, - "10": 0.22202, - "11": 0.21921, - "12": 0.21886, - "13": 0.21819, - "14": 0.21831, - "15": 0.22111, - "16": 0.22122, - "17": 0.22218, - "18": 0.22075, - "19": 0.22666, - "20": 0.2194, - "21": 0.21989, - "22": 0.21964, - "23": 0.21747, - "24": 0.21849, - "25": 0.21815, - "26": 0.21631, - "27": 0.21691, - "28": 0.21882, - "29": 0.22592, - "30": 0.22007, - "31": 0.22101, - "32": 0.21883, - "33": 0.22271, - "34": 0.2218, - "35": 0.22044, - "36": 0.21738, - "37": 0.21825, - "38": 0.21973, - "39": 0.21709, - "40": 0.21702, - "41": 0.21984, - "42": 0.22062, - "43": 0.2206, - "44": 0.21837, - "45": 0.21769, - "46": 0.21981, - "47": 0.21851, - "48": 0.21744, - "49": 0.2182, - "50": 0.21878, - "51": 0.25555, - "52": 0.26714, - "53": 0.22308, - "54": 0.22545, - "55": 0.22025, - "56": 0.22196, - "57": 0.22446, - "58": 0.21756, - "59": 0.21875, - "60": 0.21983, - "61": 0.22253, - "62": 0.2224, - "63": 0.2195, - "64": 0.221, - "65": 0.21958, - "66": 0.22081, - "67": 0.21709, - "68": 0.21941, - "69": 0.23791, - "70": 0.21957, - "71": 0.22017, - "72": 0.21767, - "73": 0.21989, - "74": 0.21807, - "75": 0.22008, - "76": 0.2181, - "77": 0.22237, - "78": 0.21893, - "79": 0.22044, - "80": 0.21745, - "81": 0.21784, - "82": 0.21951, - "83": 0.21827, - "84": 0.21882, - "85": 0.21898, - "86": 0.21833, - "87": 0.21788, - "88": 0.2192, - "89": 0.219, - "90": 0.21738, - "91": 0.21787, - "92": 0.21744, - "93": 0.2176, - "94": 0.21938, - "95": 0.22201, - "96": 0.21719, - "97": 0.21827, - "98": 0.2177, - "99": 0.21711, - "100": 0.21697 + "2": 3.20513, + "3": 0.23622, + "4": 0.24446, + "5": 0.23411, + "6": 0.23592, + "7": 0.23133, + "8": 0.23166, + "9": 0.23208, + "10": 0.23252, + "11": 0.23269, + "12": 0.23312, + "13": 0.23275, + "14": 0.23218, + "15": 0.23206, + "16": 0.23109, + "17": 0.23167, + "18": 0.23267, + "19": 0.23434, + "20": 0.23097, + "21": 0.22877, + "22": 0.22843, + "23": 0.22786, + "24": 0.23429, + "25": 0.22688, + "26": 0.23018, + "27": 0.22693, + "28": 0.22713, + "29": 0.22872, + "30": 0.22807, + "31": 0.2289, + "32": 0.22789, + "33": 0.22769, + "34": 0.22856, + "35": 0.22755, + "36": 0.22672, + "37": 0.23337, + "38": 0.22939, + "39": 0.22734, + "40": 0.229, + "41": 0.23166, + "42": 0.23029, + "43": 0.23281, + "44": 0.22951, + "45": 0.23006, + "46": 0.2287, + "47": 0.2293, + "48": 0.22851, + "49": 0.23326, + "50": 0.22881, + "51": 0.27534, + "52": 0.27804, + "53": 0.23007, + "54": 0.22967, + "55": 0.22914, + "56": 0.2435, + "57": 0.24903, + "58": 0.24366, + "59": 0.24552, + "60": 0.24848, + "61": 0.24804, + "62": 0.23436, + "63": 0.23757, + "64": 0.2425, + "65": 0.23735, + "66": 0.23057, + "67": 0.23241, + "68": 0.23077, + "69": 0.22976, + "70": 0.23097, + "71": 0.22994, + "72": 0.22854, + "73": 0.23158, + "74": 0.23335, + "75": 0.23111, + "76": 0.22927, + "77": 0.23007, + "78": 0.23671, + "79": 0.23449, + "80": 0.22918, + "81": 0.2341, + "82": 0.22931, + "83": 0.22999, + "84": 0.22921, + "85": 0.23272, + "86": 0.22978, + "87": 0.22895, + "88": 0.22886, + "89": 0.23093, + "90": 0.22965, + "91": 0.22889, + "92": 0.22874, + "93": 0.22896, + "94": 0.22963, + "95": 0.23181, + "96": 0.23007, + "97": 0.23028, + "98": 0.22981, + "99": 0.22806, + "100": 0.23091 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json index fa22b14865c..5716b655f46 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.95307, - "2": 10.94856, - "3": 10.95609, - "4": 10.94434, - "5": 10.94637, - "6": 10.94557, - "7": 10.95292, - "8": 10.93872, - "9": 10.94966, - "10": 10.93736, - "11": 10.93794, - "12": 10.93034, - "13": 10.92636, - "14": 10.92328, - "15": 10.90361, - "16": 10.89261, - "17": 10.90125, + "1": 10.95306, + "2": 10.94859, + "3": 10.95615, + "4": 10.94432, + "5": 10.94639, + "6": 10.94553, + "7": 10.95287, + "8": 10.93869, + "9": 10.94969, + "10": 10.93737, + "11": 10.93802, + "12": 10.93045, + "13": 10.92628, + "14": 10.92329, + "15": 10.90359, + "16": 10.89263, + "17": 10.90118, "18": 10.88605, - "19": 10.88664, - "20": 10.80354, - "21": 10.79325, - "22": 10.78345, - "23": 10.77589, - "24": 10.73859, - "25": 10.75397, - "26": 10.72401, - "27": 10.68087, - "28": 10.60753, - "29": 10.58173, + "19": 10.88667, + "20": 10.80348, + "21": 10.79333, + "22": 10.78352, + "23": 10.77607, + "24": 10.73853, + "25": 10.75406, + "26": 10.72412, + "27": 10.68092, + "28": 10.60761, + "29": 10.58179, "30": 10.57586, - "31": 10.55865, - "32": 10.54534, - "33": 10.51316, - "34": 10.47157, - "35": 10.47226, - "36": 10.46033, - "37": 10.41614, - "38": 10.42803, - "39": 10.38981, - "40": 10.37324, + "31": 10.5587, + "32": 10.54539, + "33": 10.51327, + "34": 10.47162, + "35": 10.47231, + "36": 10.46037, + "37": 10.41615, + "38": 10.42801, + "39": 10.38986, + "40": 10.3732, "41": 10.35698, - "42": 10.34258, - "43": 10.30625, - "44": 10.2833, - "45": 10.29373, - "46": 10.26051, - "47": 10.24268, - "48": 10.2011, - "49": 10.1927, - "50": 10.19551, - "51": 10.20263, - "52": 10.15249, + "42": 10.34265, + "43": 10.30637, + "44": 10.28335, + "45": 10.2938, + "46": 10.26052, + "47": 10.24273, + "48": 10.20119, + "49": 10.19276, + "50": 10.19553, + "51": 10.20267, + "52": 10.15253, "53": 10.15361, - "54": 10.11924, - "55": 10.09675, - "56": 10.11854, - "57": 10.10417, - "58": 10.11747, - "59": 10.06702, + "54": 10.11928, + "55": 10.09682, + "56": 10.11859, + "57": 10.10424, + "58": 10.11749, + "59": 10.06707, "60": 10.08112, - "61": 10.03586, - "62": 10.00888, - "63": 10.07618, - "64": 10.03927, - "65": 10.01139, - "66": 10.03452, - "67": 10.01084, - "68": 9.97493, - "69": 9.99636, - "70": 9.97873, - "71": 10.00559, - "72": 9.98062, + "61": 10.03592, + "62": 10.00896, + "63": 10.07624, + "64": 10.03928, + "65": 10.01145, + "66": 10.03455, + "67": 10.0109, + "68": 9.97495, + "69": 9.99643, + "70": 9.97875, + "71": 10.00566, + "72": 9.98064, "73": 9.97765, - "74": 9.95924, - "75": 9.9335, - "76": 9.96535, + "74": 9.95934, + "75": 9.93357, + "76": 9.96538, "77": 9.9598, - "78": 9.9074, - "79": 9.91725, - "80": 9.93605, - "81": 9.96051, - "82": 9.89289, - "83": 9.85645, - "84": 9.78625, - "85": 9.78401, - "86": 9.88327, - "87": 9.90989, - "88": 9.88512, - "89": 9.81694, - "90": 9.80996, - "91": 9.82887, - "92": 9.81612, - "93": 9.74684, - "94": 9.83238, - "95": 9.8273, - "96": 9.80575, - "97": 9.74064, - "98": 9.77907, - "99": 9.82087, - "100": 9.71283 + "78": 9.90744, + "79": 9.91735, + "80": 9.93615, + "81": 9.9606, + "82": 9.89295, + "83": 9.85654, + "84": 9.78626, + "85": 9.78407, + "86": 9.88333, + "87": 9.90984, + "88": 9.88523, + "89": 9.81703, + "90": 9.80999, + "91": 9.82893, + "92": 9.81614, + "93": 9.7469, + "94": 9.83241, + "95": 9.82735, + "96": 9.80576, + "97": 9.74068, + "98": 9.77912, + "99": 9.82089, + "100": 9.7129 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1879.0, - "2": 1840.0, - "3": 1763.0, - "4": 1799.0, - "5": 1826.0, - "6": 1799.0, - "7": 1948.0, - "8": 1699.0, - "9": 1844.0, - "10": 1821.0, - "11": 1760.0, - "12": 1737.0, - "13": 1900.0, - "14": 1871.0, - "15": 1586.0, - "16": 1801.0, - "17": 1831.0, - "18": 1844.0, - "19": 1719.0, - "20": 1793.0, - "21": 1923.0, - "22": 1777.0, - "23": 1785.0, - "24": 1835.0, - "25": 1854.0, - "26": 1897.0, - "27": 1952.0, - "28": 1885.0, - "29": 2125.0, - "30": 1983.0, - "31": 2149.0, - "32": 2115.0, - "33": 2121.0, - "34": 2212.0, - "35": 2218.0, - "36": 2118.0, - "37": 2349.0, - "38": 2207.0, - "39": 2256.0, - "40": 2381.0, - "41": 2353.0, - "42": 2050.0, - "43": 2518.0, - "44": 2259.0, - "45": 2659.0, - "46": 2517.0, - "47": 2552.0, - "48": 2676.0, - "49": 2840.0, - "50": 2551.0, - "51": 2495.0, - "52": 2765.0, - "53": 2808.0, - "54": 2907.0, - "55": 2675.0, - "56": 2865.0, - "57": 2242.0, - "58": 3810.0, - "59": 3052.0, - "60": 3190.0, + "1": 1873.0, + "2": 1839.0, + "3": 1794.0, + "4": 1707.0, + "5": 1848.0, + "6": 1769.0, + "7": 1956.0, + "8": 1758.0, + "9": 1923.0, + "10": 1783.0, + "11": 1753.0, + "12": 1736.0, + "13": 1729.0, + "14": 1907.0, + "15": 1662.0, + "16": 1764.0, + "17": 1859.0, + "18": 1837.0, + "19": 1818.0, + "20": 1795.0, + "21": 1814.0, + "22": 1785.0, + "23": 1794.0, + "24": 1819.0, + "25": 1851.0, + "26": 1917.0, + "27": 1897.0, + "28": 1915.0, + "29": 2081.0, + "30": 1907.0, + "31": 2059.0, + "32": 2110.0, + "33": 1994.0, + "34": 2153.0, + "35": 2197.0, + "36": 2055.0, + "37": 2273.0, + "38": 2054.0, + "39": 2186.0, + "40": 2307.0, + "41": 2398.0, + "42": 2097.0, + "43": 2504.0, + "44": 2206.0, + "45": 2602.0, + "46": 2509.0, + "47": 2512.0, + "48": 2738.0, + "49": 2728.0, + "50": 2606.0, + "51": 2591.0, + "52": 2784.0, + "53": 2689.0, + "54": 2785.0, + "55": 2711.0, + "56": 2875.0, + "57": 2338.0, + "58": 3839.0, + "59": 2922.0, + "60": 3073.0, "61": 2925.0, - "62": 3385.0, - "63": 3602.0, - "64": 3723.0, - "65": 2785.0, - "66": 3298.0, - "67": 4072.0, - "68": 3407.0, - "69": 3096.0, - "70": 3528.0, - "71": 3225.0, - "72": 3092.0, - "73": 3475.0, - "74": 3362.0, - "75": 3195.0, - "76": 3320.0, - "77": 3944.0, - "78": 3522.0, - "79": 3281.0, - "80": 3241.0, - "81": 3763.0, - "82": 2964.0, - "83": 3107.0, - "84": 3076.0, - "85": 2865.0, - "86": 2990.0, - "87": 3080.0, - "88": 3170.0, - "89": 3342.0, - "90": 3855.0, - "91": 2997.0, - "92": 2860.0, - "93": 3091.0, - "94": 3006.0, - "95": 3393.0, - "96": 3323.0, - "97": 3534.0, - "98": 3263.0, - "99": 3271.0, - "100": 3436.0 + "62": 3225.0, + "63": 3690.0, + "64": 3678.0, + "65": 2853.0, + "66": 3372.0, + "67": 4031.0, + "68": 3470.0, + "69": 3153.0, + "70": 3391.0, + "71": 3349.0, + "72": 2993.0, + "73": 3441.0, + "74": 3384.0, + "75": 3209.0, + "76": 3335.0, + "77": 3858.0, + "78": 3425.0, + "79": 3372.0, + "80": 3288.0, + "81": 3781.0, + "82": 2999.0, + "83": 3213.0, + "84": 2975.0, + "85": 2882.0, + "86": 3037.0, + "87": 3065.0, + "88": 3094.0, + "89": 3301.0, + "90": 3877.0, + "91": 3014.0, + "92": 2822.0, + "93": 3114.0, + "94": 2927.0, + "95": 3310.0, + "96": 3391.0, + "97": 3558.0, + "98": 3160.0, + "99": 3336.0, + "100": 3471.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.94756, - "3": 1.33968, - "4": 1.18099, - "5": 0.79196, - "6": 1.3329, - "7": 1.19878, - "8": 1.44642, - "9": 1.63098, - "10": 0.96334, - "11": 1.18733, - "12": 1.17451, - "13": 1.25407, - "14": 1.5443, - "15": 0.95591, - "16": 1.03101, - "17": 1.42579, - "18": 1.22868, - "19": 1.20668, - "20": 1.13172, - "21": 1.09404, - "22": 1.07889, - "23": 1.26565, - "24": 1.60129, - "25": 0.77105, - "26": 1.04269, - "27": 1.12442, - "28": 1.10423, - "29": 1.00549, - "30": 1.00908, - "31": 0.83743, - "32": 1.90219, - "33": 1.16577, - "34": 0.4742, - "35": 1.40055, - "36": 1.52214, - "37": 1.04511, - "38": 0.67596, - "39": 1.26218, - "40": 1.49779, - "41": 1.12567, - "42": 1.20837, - "43": 1.08314, - "44": 1.40335, - "45": 0.96504, - "46": 0.81372, - "47": 1.18445, - "48": 1.15653, - "49": 1.01329, - "50": 1.35972, - "51": 0.49238, - "52": 0.99614, - "53": 1.27125, - "54": 1.4193, - "55": 1.18832, - "56": 1.0429, - "57": 0.84399, - "58": 1.20729, - "59": 0.96425, - "60": 0.80519, - "61": 1.1901, - "62": 1.22612, - "63": 1.07342, - "64": 1.3296, - "65": 0.77658, - "66": 1.01715, - "67": 1.07719, - "68": 0.88113, - "69": 1.38992, - "70": 0.98348, - "71": 1.11555, - "72": 1.19298, - "73": 0.9969, - "74": 0.67233, - "75": 1.09868, - "76": 1.19409, - "77": 1.30202, - "78": 1.42778, - "79": 1.87638, - "80": 1.60954, - "81": 1.1351, - "82": 1.46354, - "83": 1.31014, - "84": 1.10204, - "85": 0.76669, - "86": 1.15568, - "87": 1.87379, - "88": 1.14966, - "89": 1.9549, - "90": 1.44848, - "91": 1.08126, - "92": 1.55667, - "93": 1.34264, - "94": 1.70687, - "95": 1.03485, - "96": 1.9428, - "97": 1.07111, - "98": 0.52931, - "99": 1.91464, - "100": 1.25041 + "2": 3.94947, + "3": 1.50396, + "4": 1.50176, + "5": 0.98181, + "6": 1.28624, + "7": 1.23151, + "8": 1.66227, + "9": 1.78298, + "10": 1.14872, + "11": 1.41917, + "12": 1.26781, + "13": 1.25132, + "14": 1.66559, + "15": 1.14181, + "16": 1.03919, + "17": 1.43587, + "18": 1.35694, + "19": 1.39136, + "20": 1.23659, + "21": 1.17551, + "22": 1.18524, + "23": 1.2885, + "24": 1.76535, + "25": 0.74425, + "26": 1.33216, + "27": 1.06919, + "28": 1.60423, + "29": 0.98875, + "30": 0.7668, + "31": 1.13699, + "32": 1.93777, + "33": 1.88581, + "34": 1.50423, + "35": 1.98384, + "36": 1.50042, + "37": 1.12102, + "38": 1.37899, + "39": 1.9987, + "40": 2.09404, + "41": 1.27729, + "42": 1.81719, + "43": 1.33548, + "44": 1.52289, + "45": 1.1413, + "46": 1.11886, + "47": 1.3187, + "48": 1.29558, + "49": 0.98724, + "50": 1.23782, + "51": 0.46871, + "52": 1.12924, + "53": 1.38464, + "54": 1.45518, + "55": 1.27735, + "56": 1.15825, + "57": 1.1152, + "58": 1.23963, + "59": 1.10146, + "60": 0.9791, + "61": 1.16944, + "62": 1.57558, + "63": 1.09616, + "64": 1.35596, + "65": 0.99194, + "66": 1.07786, + "67": 1.31415, + "68": 1.09075, + "69": 1.44256, + "70": 1.18576, + "71": 1.18382, + "72": 1.26795, + "73": 1.08208, + "74": 0.75191, + "75": 1.02329, + "76": 1.25771, + "77": 1.4109, + "78": 1.40421, + "79": 1.80777, + "80": 1.16288, + "81": 1.15847, + "82": 1.7932, + "83": 1.43956, + "84": 0.99596, + "85": 0.80696, + "86": 1.29427, + "87": 1.44706, + "88": 1.13488, + "89": 0.98534, + "90": 1.57512, + "91": 1.02928, + "92": 1.70708, + "93": 1.11789, + "94": 1.28962, + "95": 1.13981, + "96": 0.96339, + "97": 1.18812, + "98": 1.37895, + "99": 1.56866, + "100": 1.12637 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json index 4006fd0f4b8..2e0f80be1a4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89756, - "2": 10.8907, - "3": 10.88309, - "4": 10.88974, + "1": 10.89755, + "2": 10.89063, + "3": 10.88312, + "4": 10.8897, "5": 10.88254, - "6": 10.88652, - "7": 10.88675, - "8": 10.88256, - "9": 10.88485, - "10": 10.88068, - "11": 10.87902, - "12": 10.87771, - "13": 10.87243, - "14": 10.86853, - "15": 10.84593, - "16": 10.83855, - "17": 10.83759, - "18": 10.82719, - "19": 10.83322, - "20": 10.76014, - "21": 10.74126, + "6": 10.8864, + "7": 10.8867, + "8": 10.88255, + "9": 10.88479, + "10": 10.88064, + "11": 10.87912, + "12": 10.87777, + "13": 10.87248, + "14": 10.86852, + "15": 10.84599, + "16": 10.83857, + "17": 10.83762, + "18": 10.8271, + "19": 10.83323, + "20": 10.76028, + "21": 10.74135, "22": 10.72492, "23": 10.71936, - "24": 10.70189, - "25": 10.69474, - "26": 10.6776, - "27": 10.65133, - "28": 10.57392, - "29": 10.54991, - "30": 10.52269, - "31": 10.52536, - "32": 10.5134, - "33": 10.46553, - "34": 10.4375, - "35": 10.4374, - "36": 10.42118, - "37": 10.38935, - "38": 10.38941, - "39": 10.35682, - "40": 10.34743, - "41": 10.32252, - "42": 10.30529, - "43": 10.2837, - "44": 10.26289, - "45": 10.26626, - "46": 10.23668, - "47": 10.2212, - "48": 10.17262, - "49": 10.18153, - "50": 10.17787 + "24": 10.70195, + "25": 10.69471, + "26": 10.67763, + "27": 10.65143, + "28": 10.57397, + "29": 10.5501, + "30": 10.52272, + "31": 10.52542, + "32": 10.51347, + "33": 10.46558, + "34": 10.43754, + "35": 10.43746, + "36": 10.42129, + "37": 10.38943, + "38": 10.3895, + "39": 10.35679, + "40": 10.34755, + "41": 10.32261, + "42": 10.30542, + "43": 10.28368, + "44": 10.26293, + "45": 10.2663, + "46": 10.23671, + "47": 10.2213, + "48": 10.17266, + "49": 10.18158, + "50": 10.17791 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1841.0, - "2": 1782.0, - "3": 1741.0, - "4": 1707.0, - "5": 1875.0, - "6": 1802.0, - "7": 1934.0, - "8": 1730.0, - "9": 1768.0, - "10": 1790.0, - "11": 1717.0, - "12": 1854.0, - "13": 1795.0, - "14": 1901.0, - "15": 1698.0, - "16": 1856.0, - "17": 1882.0, - "18": 1857.0, - "19": 1650.0, - "20": 1778.0, - "21": 1857.0, - "22": 1827.0, - "23": 1781.0, - "24": 1845.0, - "25": 1818.0, - "26": 1877.0, - "27": 1887.0, - "28": 1888.0, + "1": 1832.0, + "2": 1769.0, + "3": 1815.0, + "4": 1867.0, + "5": 1882.0, + "6": 1755.0, + "7": 1962.0, + "8": 1704.0, + "9": 1694.0, + "10": 1718.0, + "11": 1745.0, + "12": 1800.0, + "13": 1834.0, + "14": 1964.0, + "15": 1680.0, + "16": 1817.0, + "17": 1824.0, + "18": 1753.0, + "19": 1666.0, + "20": 1759.0, + "21": 1762.0, + "22": 1742.0, + "23": 1825.0, + "24": 1895.0, + "25": 1666.0, + "26": 1897.0, + "27": 1828.0, + "28": 1800.0, "29": 1973.0, - "30": 1819.0, - "31": 2013.0, - "32": 2072.0, - "33": 2021.0, - "34": 2116.0, - "35": 2134.0, - "36": 2021.0, - "37": 2298.0, - "38": 2052.0, - "39": 2245.0, - "40": 2299.0, - "41": 2318.0, - "42": 2073.0, - "43": 2457.0, - "44": 2132.0, - "45": 2639.0, - "46": 2424.0, - "47": 2491.0, - "48": 2650.0, - "49": 2923.0, - "50": 2666.0 + "30": 1801.0, + "31": 1983.0, + "32": 1978.0, + "33": 1990.0, + "34": 2022.0, + "35": 2120.0, + "36": 2058.0, + "37": 2236.0, + "38": 2159.0, + "39": 2242.0, + "40": 2441.0, + "41": 2363.0, + "42": 2012.0, + "43": 2325.0, + "44": 2258.0, + "45": 2529.0, + "46": 2383.0, + "47": 2468.0, + "48": 2741.0, + "49": 2977.0, + "50": 2613.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.34616, - "3": 0.3142, - "4": 0.31192, - "5": 0.31604, - "6": 0.3111, - "7": 0.31244, - "8": 0.30946, - "9": 0.31192, - "10": 0.30859, - "11": 0.30942, - "12": 0.3117, - "13": 0.31095, - "14": 0.30866, - "15": 0.30981, - "16": 0.31245, - "17": 0.31087, - "18": 0.31053, - "19": 0.30871, - "20": 0.30939, - "21": 0.3118, - "22": 0.31199, - "23": 0.30966, - "24": 0.31088, - "25": 0.31307, - "26": 0.31034, - "27": 0.31087, - "28": 0.30973, - "29": 0.30976, - "30": 0.30982, - "31": 0.30984, - "32": 0.31107, - "33": 0.30863, - "34": 0.31312, - "35": 0.30891, - "36": 0.31222, - "37": 0.31222, - "38": 0.31082, - "39": 0.31106, - "40": 0.31, - "41": 0.30893, - "42": 0.30983, - "43": 0.315, - "44": 0.31091, - "45": 0.31262, - "46": 0.30891, - "47": 0.31182, - "48": 0.30917, - "49": 0.31196, - "50": 0.31219 + "2": 3.49697, + "3": 0.33407, + "4": 0.33088, + "5": 0.32882, + "6": 0.33067, + "7": 0.32949, + "8": 0.33215, + "9": 0.32871, + "10": 0.33288, + "11": 0.33269, + "12": 0.32834, + "13": 0.32695, + "14": 0.32789, + "15": 0.3269, + "16": 0.32544, + "17": 0.32784, + "18": 0.32508, + "19": 0.32745, + "20": 0.32615, + "21": 0.32818, + "22": 0.32919, + "23": 0.32741, + "24": 0.32896, + "25": 0.32756, + "26": 0.32689, + "27": 0.32654, + "28": 0.32695, + "29": 0.33194, + "30": 0.32713, + "31": 0.33026, + "32": 0.32462, + "33": 0.32758, + "34": 0.32805, + "35": 0.33259, + "36": 0.32936, + "37": 0.33182, + "38": 0.32568, + "39": 0.32797, + "40": 0.32755, + "41": 0.32868, + "42": 0.32715, + "43": 0.32867, + "44": 0.32493, + "45": 0.32655, + "46": 0.32899, + "47": 0.32649, + "48": 0.33216, + "49": 0.32775, + "50": 0.32949 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json index 63c5bf23448..7681ca255e1 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.89465, + "1": 10.89449, "2": 10.88327, - "3": 10.88075, - "4": 10.88361, - "5": 10.88287, - "6": 10.875, - "7": 10.87677, - "8": 10.88171, - "9": 10.88078, - "10": 10.87893, - "11": 10.87526, - "12": 10.86941, - "13": 10.86653, - "14": 10.86405, - "15": 10.8404, - "16": 10.83432, - "17": 10.84502, - "18": 10.81955, - "19": 10.84023, - "20": 10.75963, - "21": 10.74833, - "22": 10.73542, - "23": 10.72861, - "24": 10.70175, - "25": 10.69385, - "26": 10.68277, - "27": 10.65028, - "28": 10.58845, - "29": 10.5527, - "30": 10.53012, - "31": 10.52065, - "32": 10.51506, + "3": 10.88077, + "4": 10.88364, + "5": 10.8829, + "6": 10.87498, + "7": 10.8768, + "8": 10.88173, + "9": 10.88076, + "10": 10.87889, + "11": 10.87524, + "12": 10.86948, + "13": 10.86656, + "14": 10.86411, + "15": 10.84044, + "16": 10.8343, + "17": 10.8452, + "18": 10.81964, + "19": 10.84026, + "20": 10.75966, + "21": 10.74836, + "22": 10.73543, + "23": 10.72868, + "24": 10.70162, + "25": 10.69393, + "26": 10.68276, + "27": 10.6503, + "28": 10.58853, + "29": 10.55281, + "30": 10.53027, + "31": 10.52072, + "32": 10.51514, "33": 10.47611, - "34": 10.45066, - "35": 10.45166, - "36": 10.42459, - "37": 10.39344, - "38": 10.39768, - "39": 10.36673, - "40": 10.35691, - "41": 10.33049, - "42": 10.30995, - "43": 10.28971, - "44": 10.26685, - "45": 10.27231, - "46": 10.23941, - "47": 10.2249, - "48": 10.18112, - "49": 10.1825, - "50": 10.18095 + "34": 10.45072, + "35": 10.45162, + "36": 10.42463, + "37": 10.39354, + "38": 10.39778, + "39": 10.36681, + "40": 10.35697, + "41": 10.33059, + "42": 10.31008, + "43": 10.28977, + "44": 10.26692, + "45": 10.27237, + "46": 10.23947, + "47": 10.22497, + "48": 10.18114, + "49": 10.18252, + "50": 10.18097 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1815.0, - "2": 1778.0, - "3": 1884.0, - "4": 1723.0, - "5": 1861.0, - "6": 1834.0, - "7": 1939.0, - "8": 1787.0, - "9": 1863.0, - "10": 1827.0, - "11": 1824.0, - "12": 1776.0, - "13": 1751.0, - "14": 1921.0, - "15": 1778.0, - "16": 1802.0, - "17": 1812.0, - "18": 1776.0, - "19": 1781.0, - "20": 1731.0, - "21": 1944.0, - "22": 1875.0, - "23": 1834.0, - "24": 1839.0, - "25": 1770.0, - "26": 1904.0, - "27": 1826.0, - "28": 1782.0, - "29": 1928.0, - "30": 1923.0, - "31": 2037.0, - "32": 1915.0, - "33": 1951.0, - "34": 2059.0, - "35": 2146.0, - "36": 2031.0, - "37": 2301.0, - "38": 2103.0, - "39": 2248.0, - "40": 2226.0, - "41": 2352.0, - "42": 1925.0, - "43": 2435.0, - "44": 2241.0, - "45": 2601.0, - "46": 2495.0, - "47": 2557.0, - "48": 2705.0, - "49": 2785.0, - "50": 2545.0 + "1": 1835.0, + "2": 1740.0, + "3": 1915.0, + "4": 1845.0, + "5": 1801.0, + "6": 1751.0, + "7": 1964.0, + "8": 1798.0, + "9": 1884.0, + "10": 1854.0, + "11": 1807.0, + "12": 1799.0, + "13": 1853.0, + "14": 1976.0, + "15": 1714.0, + "16": 1875.0, + "17": 1860.0, + "18": 1818.0, + "19": 1757.0, + "20": 1785.0, + "21": 1831.0, + "22": 1790.0, + "23": 1849.0, + "24": 1850.0, + "25": 1769.0, + "26": 1857.0, + "27": 1883.0, + "28": 1851.0, + "29": 1971.0, + "30": 1913.0, + "31": 2028.0, + "32": 1980.0, + "33": 2012.0, + "34": 2034.0, + "35": 2228.0, + "36": 1947.0, + "37": 2153.0, + "38": 2099.0, + "39": 2219.0, + "40": 2291.0, + "41": 2282.0, + "42": 2009.0, + "43": 2445.0, + "44": 2251.0, + "45": 2616.0, + "46": 2503.0, + "47": 2614.0, + "48": 2743.0, + "49": 2913.0, + "50": 2590.0 } }, "mem-allocated-bytes": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21165, - "3": 2.4816, - "4": 2.18761, - "5": 1.77345, - "6": 2.25446, - "7": 1.92959, - "8": 2.58115, - "9": 2.62203, - "10": 2.16926, - "11": 2.30418, - "12": 1.94962, - "13": 2.26072, - "14": 2.32385, - "15": 1.88242, - "16": 2.10012, - "17": 2.7401, - "18": 1.92322, - "19": 2.32259, - "20": 1.92946, - "21": 2.42083, - "22": 1.79961, - "23": 2.28145, - "24": 1.79815, - "25": 2.42588, - "26": 2.61199, - "27": 1.90333, - "28": 2.00642, - "29": 1.73461, - "30": 1.99201, - "31": 1.7887, - "32": 3.05861, - "33": 1.76916, - "34": 1.65386, - "35": 2.44925, - "36": 2.34312, - "37": 1.98498, - "38": 1.83213, - "39": 2.40965, - "40": 2.02627, - "41": 2.17662, - "42": 2.70897, - "43": 1.89561, - "44": 2.08977, - "45": 2.63009, - "46": 1.8783, - "47": 2.20647, - "48": 2.38399, - "49": 1.79461, - "50": 2.29719 + "2": 5.32754, + "3": 2.54107, + "4": 2.06858, + "5": 1.78113, + "6": 2.29062, + "7": 1.82295, + "8": 2.50889, + "9": 2.54872, + "10": 2.21198, + "11": 2.32135, + "12": 1.87363, + "13": 2.31611, + "14": 2.48239, + "15": 1.87027, + "16": 2.05061, + "17": 2.57665, + "18": 1.948, + "19": 2.33851, + "20": 2.02997, + "21": 2.36121, + "22": 1.94878, + "23": 2.60279, + "24": 2.21036, + "25": 2.62965, + "26": 2.71866, + "27": 1.97404, + "28": 1.96001, + "29": 1.76424, + "30": 1.98008, + "31": 1.77878, + "32": 3.08575, + "33": 1.67473, + "34": 1.7323, + "35": 2.36159, + "36": 2.18109, + "37": 1.91671, + "38": 1.5232, + "39": 2.60447, + "40": 1.97124, + "41": 2.18503, + "42": 2.60628, + "43": 1.78469, + "44": 2.01563, + "45": 2.46075, + "46": 1.88357, + "47": 2.24842, + "48": 2.4978, + "49": 1.76617, + "50": 2.26997 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json index 33fbf30b29e..e74d7123a60 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89723, - "2": 10.89039, - "3": 10.88307, - "4": 10.88936, - "5": 10.88232, - "6": 10.88662, - "7": 10.88761, - "8": 10.88217, - "9": 10.8849, - "10": 10.88111, - "11": 10.87931, - "12": 10.87751, - "13": 10.87274, - "14": 10.8683, - "15": 10.84624, - "16": 10.83789, - "17": 10.83723, - "18": 10.82685, - "19": 10.83392, - "20": 10.76073, - "21": 10.74166, - "22": 10.72525, - "23": 10.71959, - "24": 10.70182, - "25": 10.69462, - "26": 10.67756, - "27": 10.65206, - "28": 10.57443, - "29": 10.5504, - "30": 10.52251, - "31": 10.5259, - "32": 10.51363, - "33": 10.46608, - "34": 10.4379, - "35": 10.43775, - "36": 10.42159, - "37": 10.38933, - "38": 10.38943, - "39": 10.3568, - "40": 10.34751, - "41": 10.32304, - "42": 10.30556, - "43": 10.28355, + "1": 10.89725, + "2": 10.89034, + "3": 10.88313, + "4": 10.88928, + "5": 10.8823, + "6": 10.88649, + "7": 10.88759, + "8": 10.88216, + "9": 10.88485, + "10": 10.88115, + "11": 10.87936, + "12": 10.87759, + "13": 10.87278, + "14": 10.86819, + "15": 10.84637, + "16": 10.83795, + "17": 10.8373, + "18": 10.82682, + "19": 10.83389, + "20": 10.76081, + "21": 10.74174, + "22": 10.72524, + "23": 10.71965, + "24": 10.70188, + "25": 10.69461, + "26": 10.67754, + "27": 10.6521, + "28": 10.57447, + "29": 10.55048, + "30": 10.52256, + "31": 10.52596, + "32": 10.51369, + "33": 10.46614, + "34": 10.43797, + "35": 10.43785, + "36": 10.42172, + "37": 10.38938, + "38": 10.38955, + "39": 10.35676, + "40": 10.34758, + "41": 10.32306, + "42": 10.30569, + "43": 10.28356, "44": 10.26322, - "45": 10.2662, - "46": 10.2368, - "47": 10.22146, - "48": 10.17272, - "49": 10.18156, - "50": 10.17791, - "51": 10.18023, - "52": 10.13377, - "53": 10.13153, - "54": 10.10638, - "55": 10.08296, - "56": 10.10637, - "57": 10.10201, + "45": 10.26623, + "46": 10.23684, + "47": 10.22155, + "48": 10.17277, + "49": 10.18161, + "50": 10.17798, + "51": 10.18021, + "52": 10.13376, + "53": 10.13158, + "54": 10.10647, + "55": 10.08302, + "56": 10.10644, + "57": 10.10207, "58": 10.10817, - "59": 10.04452, - "60": 10.07106, - "61": 10.02559, - "62": 9.9984, - "63": 10.06817, - "64": 10.01965, - "65": 9.98079, - "66": 10.02431, - "67": 10.00269, - "68": 9.96782, - "69": 9.98685, - "70": 9.96672, - "71": 9.99227, - "72": 9.97696, + "59": 10.04457, + "60": 10.07112, + "61": 10.0256, + "62": 9.99842, + "63": 10.06818, + "64": 10.01971, + "65": 9.98073, + "66": 10.02437, + "67": 10.0027, + "68": 9.96789, + "69": 9.98687, + "70": 9.96676, + "71": 9.9923, + "72": 9.97702, "73": 9.96099, - "74": 9.95636, - "75": 9.92492, - "76": 9.95108, - "77": 9.94711, - "78": 9.9007, - "79": 9.90293, - "80": 9.9174, - "81": 9.94305, - "82": 9.88166, - "83": 9.84046, - "84": 9.78038, - "85": 9.77116, - "86": 9.87352, - "87": 9.90184, - "88": 9.87564, - "89": 9.82672, - "90": 9.81115, - "91": 9.82071, - "92": 9.81382, - "93": 9.7493, - "94": 9.81784, - "95": 9.80819, - "96": 9.7946, - "97": 9.74391, - "98": 9.76747, - "99": 9.81868, - "100": 9.705 + "74": 9.95639, + "75": 9.92493, + "76": 9.95112, + "77": 9.94724, + "78": 9.90078, + "79": 9.90296, + "80": 9.91747, + "81": 9.94309, + "82": 9.88173, + "83": 9.84051, + "84": 9.78052, + "85": 9.7712, + "86": 9.87355, + "87": 9.9019, + "88": 9.87567, + "89": 9.82676, + "90": 9.81117, + "91": 9.82076, + "92": 9.81381, + "93": 9.74936, + "94": 9.81789, + "95": 9.80823, + "96": 9.79458, + "97": 9.74393, + "98": 9.76746, + "99": 9.81869, + "100": 9.70503 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1842.0, - "2": 1747.0, - "3": 1798.0, - "4": 1756.0, - "5": 1843.0, - "6": 1866.0, - "7": 1878.0, - "8": 1740.0, - "9": 1664.0, - "10": 1710.0, - "11": 1759.0, - "12": 1778.0, - "13": 1753.0, - "14": 1927.0, - "15": 1734.0, - "16": 1783.0, - "17": 1838.0, - "18": 1749.0, - "19": 1687.0, - "20": 1660.0, - "21": 1807.0, - "22": 1826.0, - "23": 1718.0, - "24": 1932.0, - "25": 1716.0, - "26": 1844.0, - "27": 1871.0, - "28": 1786.0, - "29": 1949.0, - "30": 1805.0, - "31": 2018.0, - "32": 2076.0, - "33": 1949.0, - "34": 2154.0, - "35": 2209.0, - "36": 2126.0, - "37": 2259.0, - "38": 2249.0, - "39": 2310.0, - "40": 2346.0, - "41": 2381.0, - "42": 2041.0, - "43": 2284.0, - "44": 2233.0, - "45": 2672.0, - "46": 2483.0, - "47": 2560.0, - "48": 2663.0, - "49": 2907.0, - "50": 2702.0, - "51": 2562.0, - "52": 2823.0, - "53": 2633.0, - "54": 2876.0, - "55": 2635.0, - "56": 2793.0, - "57": 2287.0, - "58": 3592.0, - "59": 3073.0, - "60": 2865.0, - "61": 2722.0, - "62": 3098.0, - "63": 3278.0, - "64": 3668.0, - "65": 2763.0, - "66": 3267.0, - "67": 3872.0, - "68": 3554.0, - "69": 3020.0, - "70": 3285.0, - "71": 3065.0, - "72": 3037.0, - "73": 3495.0, - "74": 3198.0, - "75": 3274.0, - "76": 3402.0, - "77": 3744.0, - "78": 3175.0, - "79": 3312.0, - "80": 3230.0, - "81": 3448.0, - "82": 2991.0, - "83": 3114.0, - "84": 2796.0, - "85": 2725.0, - "86": 3304.0, - "87": 2802.0, - "88": 3066.0, - "89": 2962.0, - "90": 3757.0, - "91": 3102.0, - "92": 2938.0, - "93": 3086.0, - "94": 3253.0, - "95": 3251.0, - "96": 3529.0, - "97": 3575.0, - "98": 3498.0, - "99": 3111.0, - "100": 3147.0 + "1": 1856.0, + "2": 1781.0, + "3": 1802.0, + "4": 1768.0, + "5": 1858.0, + "6": 1894.0, + "7": 1967.0, + "8": 1731.0, + "9": 1828.0, + "10": 1828.0, + "11": 1905.0, + "12": 1779.0, + "13": 1772.0, + "14": 1893.0, + "15": 1713.0, + "16": 1852.0, + "17": 1776.0, + "18": 1843.0, + "19": 1633.0, + "20": 1729.0, + "21": 1797.0, + "22": 1780.0, + "23": 1770.0, + "24": 1801.0, + "25": 1774.0, + "26": 1874.0, + "27": 1858.0, + "28": 1739.0, + "29": 1925.0, + "30": 1854.0, + "31": 1905.0, + "32": 2023.0, + "33": 2089.0, + "34": 2000.0, + "35": 2073.0, + "36": 2038.0, + "37": 2305.0, + "38": 2124.0, + "39": 2288.0, + "40": 2376.0, + "41": 2313.0, + "42": 1979.0, + "43": 2425.0, + "44": 2321.0, + "45": 2519.0, + "46": 2453.0, + "47": 2453.0, + "48": 2709.0, + "49": 2838.0, + "50": 2677.0, + "51": 2697.0, + "52": 2743.0, + "53": 2767.0, + "54": 2883.0, + "55": 2597.0, + "56": 2784.0, + "57": 2181.0, + "58": 3618.0, + "59": 3045.0, + "60": 2919.0, + "61": 2756.0, + "62": 3075.0, + "63": 3333.0, + "64": 3814.0, + "65": 2684.0, + "66": 3152.0, + "67": 3783.0, + "68": 3437.0, + "69": 3006.0, + "70": 3342.0, + "71": 3016.0, + "72": 3033.0, + "73": 3482.0, + "74": 3295.0, + "75": 3165.0, + "76": 3366.0, + "77": 3715.0, + "78": 3210.0, + "79": 3244.0, + "80": 3165.0, + "81": 3566.0, + "82": 3059.0, + "83": 3108.0, + "84": 2783.0, + "85": 2728.0, + "86": 3321.0, + "87": 2833.0, + "88": 2913.0, + "89": 2950.0, + "90": 3842.0, + "91": 3056.0, + "92": 2852.0, + "93": 3157.0, + "94": 3313.0, + "95": 3216.0, + "96": 3504.0, + "97": 3442.0, + "98": 3310.0, + "99": 3087.0, + "100": 3154.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.68206, - "3": 0.26799, - "4": 0.2671, - "5": 0.26641, - "6": 0.31133, - "7": 0.27562, - "8": 0.26427, - "9": 0.26467, - "10": 0.26681, - "11": 0.27792, - "12": 0.26647, - "13": 0.26615, - "14": 0.26504, - "15": 0.264, - "16": 0.26697, - "17": 0.26795, - "18": 0.26802, - "19": 0.2715, - "20": 0.26665, - "21": 0.26474, - "22": 0.26577, - "23": 0.26592, - "24": 0.27183, - "25": 0.27729, - "26": 0.27451, - "27": 0.27043, - "28": 0.27215, - "29": 0.27302, - "30": 0.27219, - "31": 0.27309, - "32": 0.27237, - "33": 0.26761, - "34": 0.26582, - "35": 0.26487, - "36": 0.26491, - "37": 0.26612, - "38": 0.26506, - "39": 0.26657, - "40": 0.26566, - "41": 0.26382, - "42": 0.26404, - "43": 0.26286, - "44": 0.27058, - "45": 0.26205, - "46": 0.26569, - "47": 0.26545, - "48": 0.26636, - "49": 0.26479, - "50": 0.26649, - "51": 0.30944, - "52": 0.31633, - "53": 0.26494, - "54": 0.26768, - "55": 0.26484, - "56": 0.27388, - "57": 0.26887, - "58": 0.27093, - "59": 0.26881, - "60": 0.26864, - "61": 0.27071, - "62": 0.268, - "63": 0.2657, - "64": 0.26699, - "65": 0.26655, - "66": 0.26481, - "67": 0.26794, - "68": 0.26571, - "69": 0.26645, - "70": 0.26652, - "71": 0.26664, - "72": 0.26645, - "73": 0.26751, - "74": 0.26629, - "75": 0.26542, - "76": 0.26591, - "77": 0.26716, - "78": 0.26552, - "79": 0.27067, - "80": 0.26779, - "81": 0.2681, - "82": 0.26482, - "83": 0.26343, - "84": 0.26212, - "85": 0.26322, - "86": 0.26917, - "87": 0.26612, - "88": 0.26696, - "89": 0.26336, - "90": 0.26479, - "91": 0.26292, - "92": 0.26877, - "93": 0.28095, - "94": 0.26692, - "95": 0.26419, - "96": 0.26691, - "97": 0.26422, - "98": 0.26459, - "99": 0.2646, - "100": 0.26515 + "2": 4.47029, + "3": 0.29353, + "4": 0.28753, + "5": 0.29543, + "6": 0.29527, + "7": 0.28919, + "8": 0.28614, + "9": 0.28851, + "10": 0.2888, + "11": 0.28974, + "12": 0.28942, + "13": 0.28913, + "14": 0.2908, + "15": 0.29368, + "16": 0.29013, + "17": 0.28828, + "18": 0.29021, + "19": 0.29199, + "20": 0.29037, + "21": 0.29002, + "22": 0.29557, + "23": 0.2895, + "24": 0.29198, + "25": 0.29525, + "26": 0.28854, + "27": 0.28922, + "28": 0.2901, + "29": 0.29786, + "30": 0.28962, + "31": 0.28945, + "32": 0.29539, + "33": 0.28937, + "34": 0.29545, + "35": 0.2917, + "36": 0.29454, + "37": 0.29226, + "38": 0.29008, + "39": 0.30005, + "40": 0.28853, + "41": 0.28803, + "42": 0.28877, + "43": 0.2911, + "44": 0.28944, + "45": 0.28766, + "46": 0.28999, + "47": 0.29613, + "48": 0.29091, + "49": 0.29011, + "50": 0.28931, + "51": 0.34831, + "52": 0.33928, + "53": 0.29041, + "54": 0.2876, + "55": 0.28968, + "56": 0.28802, + "57": 0.28934, + "58": 0.28899, + "59": 0.29244, + "60": 0.29009, + "61": 0.29456, + "62": 0.29166, + "63": 0.29001, + "64": 0.29035, + "65": 0.29649, + "66": 0.29058, + "67": 0.29064, + "68": 0.28941, + "69": 0.29168, + "70": 0.28826, + "71": 0.28735, + "72": 0.28799, + "73": 0.28951, + "74": 0.28958, + "75": 0.29054, + "76": 0.29129, + "77": 0.29473, + "78": 0.28878, + "79": 0.28763, + "80": 0.28814, + "81": 0.28612, + "82": 0.29416, + "83": 0.29107, + "84": 0.28534, + "85": 0.28605, + "86": 0.28744, + "87": 0.28613, + "88": 0.28761, + "89": 0.28746, + "90": 0.28744, + "91": 0.28988, + "92": 0.29029, + "93": 0.2858, + "94": 0.28807, + "95": 0.28686, + "96": 0.29226, + "97": 0.28789, + "98": 0.28776, + "99": 0.29078, + "100": 0.28695 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json index 8d2c2b58d58..d0933fcb57d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89463, - "2": 10.88332, - "3": 10.88064, - "4": 10.88309, - "5": 10.88332, - "6": 10.87492, - "7": 10.87689, - "8": 10.88182, - "9": 10.88055, - "10": 10.879, - "11": 10.8752, + "1": 10.89457, + "2": 10.8833, + "3": 10.88066, + "4": 10.88313, + "5": 10.88335, + "6": 10.87485, + "7": 10.87691, + "8": 10.88184, + "9": 10.88054, + "10": 10.87894, + "11": 10.87523, "12": 10.87014, - "13": 10.86618, - "14": 10.86433, - "15": 10.83999, - "16": 10.83442, - "17": 10.84483, - "18": 10.81981, - "19": 10.84023, - "20": 10.75974, - "21": 10.74842, - "22": 10.73585, - "23": 10.72846, - "24": 10.70204, - "25": 10.69386, - "26": 10.68294, - "27": 10.65042, - "28": 10.58861, - "29": 10.55276, - "30": 10.52991, - "31": 10.52067, - "32": 10.51487, - "33": 10.47672, - "34": 10.45098, - "35": 10.45204, - "36": 10.42467, - "37": 10.3935, - "38": 10.39796, - "39": 10.36701, - "40": 10.35717, - "41": 10.33082, - "42": 10.31015, - "43": 10.28972, - "44": 10.26722, - "45": 10.27276, - "46": 10.23956, - "47": 10.22509, + "13": 10.86616, + "14": 10.86431, + "15": 10.84011, + "16": 10.83443, + "17": 10.84496, + "18": 10.81997, + "19": 10.84015, + "20": 10.75975, + "21": 10.74843, + "22": 10.73591, + "23": 10.72848, + "24": 10.70201, + "25": 10.69395, + "26": 10.68291, + "27": 10.65044, + "28": 10.58865, + "29": 10.5528, + "30": 10.53001, + "31": 10.52071, + "32": 10.51493, + "33": 10.47669, + "34": 10.45106, + "35": 10.45202, + "36": 10.42475, + "37": 10.39359, + "38": 10.39804, + "39": 10.36707, + "40": 10.35722, + "41": 10.33091, + "42": 10.31022, + "43": 10.28973, + "44": 10.26728, + "45": 10.27285, + "46": 10.23962, + "47": 10.22516, "48": 10.18113, - "49": 10.18221, - "50": 10.18109, - "51": 10.18054, - "52": 10.13848, - "53": 10.14381, - "54": 10.10815, - "55": 10.08196, - "56": 10.10878, - "57": 10.09685, + "49": 10.18226, + "50": 10.1811, + "51": 10.18062, + "52": 10.13852, + "53": 10.14386, + "54": 10.10818, + "55": 10.08201, + "56": 10.10885, + "57": 10.09687, "58": 10.11263, - "59": 10.06067, + "59": 10.06068, "60": 10.08081, - "61": 10.029, - "62": 10.00208, - "63": 10.07351, - "64": 10.02973, - "65": 10.00767, - "66": 10.02968, - "67": 10.00724, - "68": 9.96811, - "69": 9.99247, - "70": 9.96962, - "71": 9.99959, - "72": 9.97932, + "61": 10.02902, + "62": 10.00206, + "63": 10.07353, + "64": 10.02981, + "65": 10.0077, + "66": 10.02965, + "67": 10.00722, + "68": 9.96815, + "69": 9.99249, + "70": 9.96967, + "71": 9.99962, + "72": 9.97935, "73": 9.97019, - "74": 9.9547, - "75": 9.93058, - "76": 9.96382, - "77": 9.95341, - "78": 9.90812, - "79": 9.91503, - "80": 9.92521, - "81": 9.95289, - "82": 9.88577, - "83": 9.85328, - "84": 9.79572, - "85": 9.78589, - "86": 9.87801, - "87": 9.90219, - "88": 9.87507, - "89": 9.82262, - "90": 9.81115, - "91": 9.82642, - "92": 9.81572, - "93": 9.75371, - "94": 9.82278, - "95": 9.81996, - "96": 9.80283, - "97": 9.74847, - "98": 9.77372, - "99": 9.81917, - "100": 9.71201 + "74": 9.95472, + "75": 9.93064, + "76": 9.96386, + "77": 9.95355, + "78": 9.90822, + "79": 9.91513, + "80": 9.92528, + "81": 9.95294, + "82": 9.88587, + "83": 9.85333, + "84": 9.79576, + "85": 9.78595, + "86": 9.87805, + "87": 9.90221, + "88": 9.87509, + "89": 9.82267, + "90": 9.81121, + "91": 9.82646, + "92": 9.81581, + "93": 9.75374, + "94": 9.82283, + "95": 9.81998, + "96": 9.80288, + "97": 9.74858, + "98": 9.77376, + "99": 9.81918, + "100": 9.71205 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1922.0, - "2": 1807.0, - "3": 1882.0, - "4": 1775.0, - "5": 1912.0, - "6": 1714.0, - "7": 1974.0, - "8": 1935.0, - "9": 1833.0, - "10": 1932.0, - "11": 1923.0, - "12": 1769.0, - "13": 1796.0, - "14": 2001.0, - "15": 1767.0, - "16": 1851.0, - "17": 1834.0, - "18": 1796.0, - "19": 1749.0, - "20": 1655.0, - "21": 1824.0, - "22": 1766.0, - "23": 1820.0, - "24": 1931.0, - "25": 1746.0, - "26": 1895.0, - "27": 1859.0, - "28": 1860.0, - "29": 1939.0, - "30": 1870.0, - "31": 2013.0, - "32": 1982.0, - "33": 2040.0, - "34": 2102.0, + "1": 1834.0, + "2": 1741.0, + "3": 1887.0, + "4": 1793.0, + "5": 1911.0, + "6": 1830.0, + "7": 1956.0, + "8": 1826.0, + "9": 1873.0, + "10": 1824.0, + "11": 1830.0, + "12": 1861.0, + "13": 1810.0, + "14": 1928.0, + "15": 1787.0, + "16": 1797.0, + "17": 1926.0, + "18": 1868.0, + "19": 1734.0, + "20": 1707.0, + "21": 2002.0, + "22": 1769.0, + "23": 1784.0, + "24": 1993.0, + "25": 1663.0, + "26": 1799.0, + "27": 1819.0, + "28": 1793.0, + "29": 1978.0, + "30": 1846.0, + "31": 2065.0, + "32": 1998.0, + "33": 1969.0, + "34": 2030.0, "35": 2075.0, - "36": 2026.0, - "37": 2314.0, - "38": 2141.0, - "39": 2189.0, - "40": 2255.0, - "41": 2396.0, - "42": 1953.0, - "43": 2389.0, - "44": 2234.0, - "45": 2588.0, - "46": 2583.0, - "47": 2581.0, - "48": 2757.0, - "49": 2847.0, - "50": 2589.0, - "51": 2438.0, - "52": 2832.0, - "53": 2666.0, - "54": 2900.0, - "55": 2729.0, - "56": 2824.0, - "57": 2182.0, - "58": 3842.0, - "59": 2971.0, - "60": 3042.0, - "61": 2689.0, - "62": 3269.0, - "63": 3494.0, - "64": 3553.0, - "65": 2767.0, - "66": 3146.0, - "67": 3952.0, - "68": 3552.0, - "69": 3054.0, - "70": 3377.0, - "71": 3173.0, - "72": 2909.0, - "73": 3460.0, - "74": 3310.0, - "75": 3240.0, - "76": 3197.0, - "77": 3644.0, - "78": 3322.0, - "79": 3270.0, - "80": 3197.0, - "81": 3406.0, - "82": 2921.0, - "83": 2929.0, - "84": 2873.0, - "85": 2722.0, - "86": 3294.0, - "87": 2854.0, - "88": 3109.0, - "89": 3067.0, - "90": 3859.0, - "91": 2975.0, - "92": 3083.0, - "93": 3074.0, - "94": 3298.0, - "95": 3271.0, - "96": 3531.0, - "97": 3566.0, - "98": 3646.0, - "99": 3246.0, - "100": 3560.0 + "36": 2019.0, + "37": 2275.0, + "38": 2039.0, + "39": 2182.0, + "40": 2303.0, + "41": 2367.0, + "42": 2060.0, + "43": 2514.0, + "44": 2224.0, + "45": 2665.0, + "46": 2587.0, + "47": 2608.0, + "48": 2761.0, + "49": 2875.0, + "50": 2566.0, + "51": 2495.0, + "52": 2745.0, + "53": 2716.0, + "54": 2941.0, + "55": 2646.0, + "56": 2783.0, + "57": 2219.0, + "58": 3749.0, + "59": 3080.0, + "60": 2948.0, + "61": 2729.0, + "62": 3297.0, + "63": 3479.0, + "64": 3568.0, + "65": 2790.0, + "66": 3159.0, + "67": 3953.0, + "68": 3534.0, + "69": 3111.0, + "70": 3394.0, + "71": 3076.0, + "72": 3103.0, + "73": 3502.0, + "74": 3458.0, + "75": 3260.0, + "76": 3172.0, + "77": 3647.0, + "78": 3314.0, + "79": 3252.0, + "80": 3179.0, + "81": 3383.0, + "82": 2774.0, + "83": 3012.0, + "84": 2861.0, + "85": 2746.0, + "86": 3245.0, + "87": 2921.0, + "88": 3137.0, + "89": 3101.0, + "90": 3855.0, + "91": 3068.0, + "92": 2903.0, + "93": 3124.0, + "94": 3341.0, + "95": 3345.0, + "96": 3457.0, + "97": 3584.0, + "98": 3644.0, + "99": 3274.0, + "100": 3526.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.5611, - "3": 3.12672, - "4": 2.50292, - "5": 2.11785, - "6": 2.60046, - "7": 2.21771, - "8": 3.05636, - "9": 3.1505, - "10": 2.70528, - "11": 3.00452, - "12": 2.61449, - "13": 2.82307, - "14": 3.18549, - "15": 2.47155, - "16": 2.67596, - "17": 3.55548, - "18": 2.7566, - "19": 2.92395, - "20": 2.62166, - "21": 2.92505, - "22": 2.36191, - "23": 2.89319, - "24": 2.37961, - "25": 3.02937, - "26": 3.18521, - "27": 2.63733, - "28": 3.15102, - "29": 3.36992, - "30": 3.93325, - "31": 3.37954, - "32": 6.69137, - "33": 3.03178, - "34": 2.9217, - "35": 3.51875, - "36": 3.67603, - "37": 2.90199, - "38": 2.57109, - "39": 4.10516, - "40": 3.70257, - "41": 3.95061, - "42": 3.62745, - "43": 2.46073, - "44": 2.61556, - "45": 3.35226, - "46": 2.33188, - "47": 2.38682, - "48": 3.00986, - "49": 2.16372, - "50": 2.58813, - "51": 2.2218, - "52": 3.04153, - "53": 2.41338, - "54": 2.93755, - "55": 2.45756, - "56": 3.56881, - "57": 2.0903, - "58": 3.0333, - "59": 2.50846, - "60": 2.4666, - "61": 3.10107, - "62": 3.03738, - "63": 2.75714, - "64": 2.71472, - "65": 2.50203, - "66": 2.69262, - "67": 2.78985, - "68": 1.8342, - "69": 2.59825, - "70": 2.26667, - "71": 2.84191, - "72": 2.27394, - "73": 2.40866, - "74": 2.20699, - "75": 2.31441, - "76": 2.50752, - "77": 2.35908, - "78": 2.4069, - "79": 3.1483, - "80": 2.8069, - "81": 3.01487, - "82": 2.9794, - "83": 3.03638, - "84": 2.45697, - "85": 3.09786, - "86": 2.35509, - "87": 2.9868, - "88": 2.69431, - "89": 2.69174, - "90": 2.54919, - "91": 2.48305, - "92": 2.8429, - "93": 2.0244, - "94": 2.53131, - "95": 2.49735, - "96": 2.50347, - "97": 2.23901, - "98": 2.39175, - "99": 2.89921, - "100": 2.06984 + "2": 5.10002, + "3": 2.47529, + "4": 2.02832, + "5": 1.7331, + "6": 2.16537, + "7": 1.78315, + "8": 2.60724, + "9": 2.50085, + "10": 2.1727, + "11": 2.29739, + "12": 1.99728, + "13": 2.22642, + "14": 2.39415, + "15": 1.81113, + "16": 2.02127, + "17": 2.73951, + "18": 2.01441, + "19": 2.19818, + "20": 1.9892, + "21": 2.29613, + "22": 1.93477, + "23": 2.34194, + "24": 1.90505, + "25": 2.4806, + "26": 2.63279, + "27": 2.05764, + "28": 2.02491, + "29": 1.75133, + "30": 2.0426, + "31": 1.69912, + "32": 2.77981, + "33": 1.68937, + "34": 1.59939, + "35": 2.38474, + "36": 2.20431, + "37": 1.93674, + "38": 1.6347, + "39": 2.38146, + "40": 1.94763, + "41": 2.1781, + "42": 2.67619, + "43": 1.90339, + "44": 1.98766, + "45": 2.5379, + "46": 1.92194, + "47": 2.17499, + "48": 2.59267, + "49": 1.83853, + "50": 2.31577, + "51": 1.9558, + "52": 2.28329, + "53": 2.10643, + "54": 2.43234, + "55": 2.09937, + "56": 2.77998, + "57": 1.5257, + "58": 2.25081, + "59": 2.00548, + "60": 1.89602, + "61": 2.26551, + "62": 2.60887, + "63": 1.68391, + "64": 2.23257, + "65": 1.93414, + "66": 2.13986, + "67": 2.27041, + "68": 1.53662, + "69": 2.11509, + "70": 1.92482, + "71": 2.45452, + "72": 1.92037, + "73": 1.99766, + "74": 1.82424, + "75": 1.78865, + "76": 2.05018, + "77": 2.01437, + "78": 2.14854, + "79": 2.4907, + "80": 2.17565, + "81": 2.54592, + "82": 2.51819, + "83": 2.29239, + "84": 1.93467, + "85": 2.22116, + "86": 2.40164, + "87": 2.95773, + "88": 2.47527, + "89": 2.3051, + "90": 2.19092, + "91": 2.38249, + "92": 2.71741, + "93": 1.96049, + "94": 2.41831, + "95": 2.21455, + "96": 2.21377, + "97": 1.89249, + "98": 2.13171, + "99": 2.73081, + "100": 1.95971 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json index 30cbdd56c51..98521fae2e8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89723, - "2": 10.89039, - "3": 10.88307, - "4": 10.88936, - "5": 10.88232, - "6": 10.88662, - "7": 10.88761, - "8": 10.88217, - "9": 10.8849, - "10": 10.88111, - "11": 10.87931, - "12": 10.87751, - "13": 10.87274, - "14": 10.8683, - "15": 10.84624, - "16": 10.83789, - "17": 10.83723, - "18": 10.82685, - "19": 10.83392, - "20": 10.76073, - "21": 10.74166, - "22": 10.72525, - "23": 10.71959, - "24": 10.70182, - "25": 10.69462, - "26": 10.67756, - "27": 10.65206, - "28": 10.57443, - "29": 10.5504, - "30": 10.52251, - "31": 10.5259, - "32": 10.51363, - "33": 10.46608, - "34": 10.4379, - "35": 10.43775, - "36": 10.42159, - "37": 10.38933, - "38": 10.38943, - "39": 10.3568, - "40": 10.34751, - "41": 10.32304, - "42": 10.30556, - "43": 10.28355, + "1": 10.89725, + "2": 10.89034, + "3": 10.88313, + "4": 10.88928, + "5": 10.8823, + "6": 10.88649, + "7": 10.88759, + "8": 10.88216, + "9": 10.88485, + "10": 10.88115, + "11": 10.87936, + "12": 10.87759, + "13": 10.87278, + "14": 10.86819, + "15": 10.84637, + "16": 10.83795, + "17": 10.8373, + "18": 10.82682, + "19": 10.83389, + "20": 10.76081, + "21": 10.74174, + "22": 10.72524, + "23": 10.71965, + "24": 10.70188, + "25": 10.69461, + "26": 10.67754, + "27": 10.6521, + "28": 10.57447, + "29": 10.55048, + "30": 10.52256, + "31": 10.52596, + "32": 10.51369, + "33": 10.46614, + "34": 10.43797, + "35": 10.43785, + "36": 10.42172, + "37": 10.38938, + "38": 10.38955, + "39": 10.35676, + "40": 10.34758, + "41": 10.32306, + "42": 10.30569, + "43": 10.28356, "44": 10.26322, - "45": 10.2662, - "46": 10.2368, - "47": 10.22146, - "48": 10.17272, - "49": 10.18156, - "50": 10.17791, - "51": 10.18023, - "52": 10.13377, - "53": 10.13153, - "54": 10.10638, - "55": 10.08296, - "56": 10.10637, - "57": 10.10201, + "45": 10.26623, + "46": 10.23684, + "47": 10.22155, + "48": 10.17277, + "49": 10.18161, + "50": 10.17798, + "51": 10.18021, + "52": 10.13376, + "53": 10.13158, + "54": 10.10647, + "55": 10.08302, + "56": 10.10644, + "57": 10.10207, "58": 10.10817, - "59": 10.04452, - "60": 10.07106, - "61": 10.02559, - "62": 9.9984, - "63": 10.06817, - "64": 10.01965, - "65": 9.98079, - "66": 10.02431, - "67": 10.00269, - "68": 9.96782, - "69": 9.98685, - "70": 9.96672, - "71": 9.99227, - "72": 9.97696, + "59": 10.04457, + "60": 10.07112, + "61": 10.0256, + "62": 9.99842, + "63": 10.06818, + "64": 10.01971, + "65": 9.98073, + "66": 10.02437, + "67": 10.0027, + "68": 9.96789, + "69": 9.98687, + "70": 9.96676, + "71": 9.9923, + "72": 9.97702, "73": 9.96099, - "74": 9.95636, - "75": 9.92492, - "76": 9.95108, - "77": 9.94711, - "78": 9.9007, - "79": 9.90293, - "80": 9.9174, - "81": 9.94305, - "82": 9.88166, - "83": 9.84046, - "84": 9.78038, - "85": 9.77116, - "86": 9.87352, - "87": 9.90184, - "88": 9.87564, - "89": 9.82672, - "90": 9.81115, - "91": 9.82071, - "92": 9.81382, - "93": 9.7493, - "94": 9.81784, - "95": 9.80819, - "96": 9.7946, - "97": 9.74391, - "98": 9.76747, - "99": 9.81868, - "100": 9.705 + "74": 9.95639, + "75": 9.92493, + "76": 9.95112, + "77": 9.94724, + "78": 9.90078, + "79": 9.90296, + "80": 9.91747, + "81": 9.94309, + "82": 9.88173, + "83": 9.84051, + "84": 9.78052, + "85": 9.7712, + "86": 9.87355, + "87": 9.9019, + "88": 9.87567, + "89": 9.82676, + "90": 9.81117, + "91": 9.82076, + "92": 9.81381, + "93": 9.74936, + "94": 9.81789, + "95": 9.80823, + "96": 9.79458, + "97": 9.74393, + "98": 9.76746, + "99": 9.81869, + "100": 9.70503 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1842.0, - "2": 1747.0, - "3": 1798.0, - "4": 1756.0, - "5": 1843.0, - "6": 1866.0, - "7": 1878.0, - "8": 1740.0, - "9": 1664.0, - "10": 1710.0, - "11": 1759.0, - "12": 1778.0, - "13": 1753.0, - "14": 1927.0, - "15": 1734.0, - "16": 1783.0, - "17": 1838.0, - "18": 1749.0, - "19": 1687.0, - "20": 1660.0, - "21": 1807.0, - "22": 1826.0, - "23": 1718.0, - "24": 1932.0, - "25": 1716.0, - "26": 1844.0, - "27": 1871.0, - "28": 1786.0, - "29": 1949.0, - "30": 1805.0, - "31": 2018.0, - "32": 2076.0, - "33": 1949.0, - "34": 2154.0, - "35": 2209.0, - "36": 2126.0, - "37": 2259.0, - "38": 2249.0, - "39": 2310.0, - "40": 2346.0, - "41": 2381.0, - "42": 2041.0, - "43": 2284.0, - "44": 2233.0, - "45": 2672.0, - "46": 2483.0, - "47": 2560.0, - "48": 2663.0, - "49": 2907.0, - "50": 2702.0, - "51": 2562.0, - "52": 2823.0, - "53": 2633.0, - "54": 2876.0, - "55": 2635.0, - "56": 2793.0, - "57": 2287.0, - "58": 3592.0, - "59": 3073.0, - "60": 2865.0, - "61": 2722.0, - "62": 3098.0, - "63": 3278.0, - "64": 3668.0, - "65": 2763.0, - "66": 3267.0, - "67": 3872.0, - "68": 3554.0, - "69": 3020.0, - "70": 3285.0, - "71": 3065.0, - "72": 3037.0, - "73": 3495.0, - "74": 3198.0, - "75": 3274.0, - "76": 3402.0, - "77": 3744.0, - "78": 3175.0, - "79": 3312.0, - "80": 3230.0, - "81": 3448.0, - "82": 2991.0, - "83": 3114.0, - "84": 2796.0, - "85": 2725.0, - "86": 3304.0, - "87": 2802.0, - "88": 3066.0, - "89": 2962.0, - "90": 3757.0, - "91": 3102.0, - "92": 2938.0, - "93": 3086.0, - "94": 3253.0, - "95": 3251.0, - "96": 3529.0, - "97": 3575.0, - "98": 3498.0, - "99": 3111.0, - "100": 3147.0 + "1": 1856.0, + "2": 1781.0, + "3": 1802.0, + "4": 1768.0, + "5": 1858.0, + "6": 1894.0, + "7": 1967.0, + "8": 1731.0, + "9": 1828.0, + "10": 1828.0, + "11": 1905.0, + "12": 1779.0, + "13": 1772.0, + "14": 1893.0, + "15": 1713.0, + "16": 1852.0, + "17": 1776.0, + "18": 1843.0, + "19": 1633.0, + "20": 1729.0, + "21": 1797.0, + "22": 1780.0, + "23": 1770.0, + "24": 1801.0, + "25": 1774.0, + "26": 1874.0, + "27": 1858.0, + "28": 1739.0, + "29": 1925.0, + "30": 1854.0, + "31": 1905.0, + "32": 2023.0, + "33": 2089.0, + "34": 2000.0, + "35": 2073.0, + "36": 2038.0, + "37": 2305.0, + "38": 2124.0, + "39": 2288.0, + "40": 2376.0, + "41": 2313.0, + "42": 1979.0, + "43": 2425.0, + "44": 2321.0, + "45": 2519.0, + "46": 2453.0, + "47": 2453.0, + "48": 2709.0, + "49": 2838.0, + "50": 2677.0, + "51": 2697.0, + "52": 2743.0, + "53": 2767.0, + "54": 2883.0, + "55": 2597.0, + "56": 2784.0, + "57": 2181.0, + "58": 3618.0, + "59": 3045.0, + "60": 2919.0, + "61": 2756.0, + "62": 3075.0, + "63": 3333.0, + "64": 3814.0, + "65": 2684.0, + "66": 3152.0, + "67": 3783.0, + "68": 3437.0, + "69": 3006.0, + "70": 3342.0, + "71": 3016.0, + "72": 3033.0, + "73": 3482.0, + "74": 3295.0, + "75": 3165.0, + "76": 3366.0, + "77": 3715.0, + "78": 3210.0, + "79": 3244.0, + "80": 3165.0, + "81": 3566.0, + "82": 3059.0, + "83": 3108.0, + "84": 2783.0, + "85": 2728.0, + "86": 3321.0, + "87": 2833.0, + "88": 2913.0, + "89": 2950.0, + "90": 3842.0, + "91": 3056.0, + "92": 2852.0, + "93": 3157.0, + "94": 3313.0, + "95": 3216.0, + "96": 3504.0, + "97": 3442.0, + "98": 3310.0, + "99": 3087.0, + "100": 3154.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.09229, - "3": 0.28973, - "4": 0.28117, - "5": 0.28002, - "6": 0.28054, - "7": 0.28075, - "8": 0.30829, - "9": 0.29364, - "10": 0.29581, - "11": 0.30981, - "12": 0.30276, - "13": 0.28494, - "14": 0.28073, - "15": 0.28508, - "16": 0.28287, - "17": 0.28038, - "18": 0.28078, - "19": 0.28009, - "20": 0.28575, - "21": 0.2791, - "22": 0.28316, - "23": 0.28062, - "24": 0.2796, - "25": 0.27874, - "26": 0.28657, - "27": 0.2803, - "28": 0.28039, - "29": 0.28157, - "30": 0.2804, - "31": 0.27957, - "32": 0.2786, - "33": 0.27909, - "34": 0.27978, - "35": 0.28196, - "36": 0.28427, - "37": 0.27865, - "38": 0.27838, - "39": 0.28176, - "40": 0.28164, - "41": 0.28141, - "42": 0.27817, - "43": 0.28011, - "44": 0.28334, - "45": 0.27989, - "46": 0.28069, - "47": 0.27997, - "48": 0.2789, - "49": 0.27814, - "50": 0.28065, - "51": 0.31687, - "52": 0.33462, - "53": 0.28535, - "54": 0.28977, - "55": 0.28138, - "56": 0.2793, - "57": 0.28052, - "58": 0.28126, - "59": 0.2805, - "60": 0.28272, - "61": 0.28265, - "62": 0.28024, - "63": 0.28263, - "64": 0.27999, - "65": 0.28232, - "66": 0.28135, - "67": 0.28159, - "68": 0.28314, - "69": 0.28021, - "70": 0.28044, - "71": 0.28249, - "72": 0.28144, - "73": 0.28031, - "74": 0.28248, - "75": 0.28182, - "76": 0.28026, - "77": 0.27954, - "78": 0.27843, - "79": 0.28015, - "80": 0.27921, - "81": 0.27873, - "82": 0.27856, - "83": 0.27898, - "84": 0.27835, - "85": 0.27764, - "86": 0.27869, - "87": 0.2796, - "88": 0.28004, - "89": 0.28243, - "90": 0.27895, - "91": 0.28368, - "92": 0.27711, - "93": 0.27854, - "94": 0.28103, - "95": 0.27794, - "96": 0.27842, - "97": 0.2811, - "98": 0.28651, - "99": 0.27871, - "100": 0.28202 + "2": 4.09232, + "3": 0.29397, + "4": 0.2942, + "5": 0.29513, + "6": 0.2921, + "7": 0.28951, + "8": 0.29541, + "9": 0.29046, + "10": 0.28975, + "11": 0.28937, + "12": 0.29081, + "13": 0.29082, + "14": 0.29484, + "15": 0.29309, + "16": 0.29178, + "17": 0.29283, + "18": 0.28964, + "19": 0.28821, + "20": 0.29278, + "21": 0.2899, + "22": 0.28889, + "23": 0.29036, + "24": 0.29086, + "25": 0.29416, + "26": 0.2913, + "27": 0.29207, + "28": 0.29292, + "29": 0.29036, + "30": 0.29297, + "31": 0.29022, + "32": 0.2926, + "33": 0.2916, + "34": 0.29296, + "35": 0.29496, + "36": 0.29269, + "37": 0.29094, + "38": 0.29029, + "39": 0.29038, + "40": 0.29135, + "41": 0.29164, + "42": 0.29053, + "43": 0.29193, + "44": 0.28802, + "45": 0.29114, + "46": 0.2902, + "47": 0.29362, + "48": 0.29534, + "49": 0.29097, + "50": 0.29191, + "51": 0.34267, + "52": 0.36049, + "53": 0.30288, + "54": 0.2985, + "55": 0.29883, + "56": 0.29616, + "57": 0.30066, + "58": 0.30281, + "59": 0.29567, + "60": 0.29659, + "61": 0.29631, + "62": 0.29586, + "63": 0.2973, + "64": 0.29654, + "65": 0.29627, + "66": 0.2953, + "67": 0.30089, + "68": 0.30109, + "69": 0.29692, + "70": 0.29761, + "71": 0.2955, + "72": 0.29845, + "73": 0.29758, + "74": 0.29669, + "75": 0.29418, + "76": 0.2939, + "77": 0.29807, + "78": 0.29716, + "79": 0.29889, + "80": 0.29708, + "81": 0.29579, + "82": 0.29511, + "83": 0.29247, + "84": 0.29489, + "85": 0.29484, + "86": 0.29739, + "87": 0.29584, + "88": 0.29986, + "89": 0.29668, + "90": 0.29666, + "91": 0.30393, + "92": 0.29634, + "93": 0.29528, + "94": 0.29763, + "95": 0.297, + "96": 0.30032, + "97": 0.29687, + "98": 0.29968, + "99": 0.29866, + "100": 0.29691 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json index 0bd2ea0961b..c23952f3b95 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89463, - "2": 10.88332, - "3": 10.88064, - "4": 10.88309, - "5": 10.88332, - "6": 10.87492, - "7": 10.87689, - "8": 10.88182, - "9": 10.88055, - "10": 10.879, - "11": 10.8752, + "1": 10.89457, + "2": 10.8833, + "3": 10.88066, + "4": 10.88313, + "5": 10.88335, + "6": 10.87485, + "7": 10.87691, + "8": 10.88184, + "9": 10.88054, + "10": 10.87894, + "11": 10.87523, "12": 10.87014, - "13": 10.86618, - "14": 10.86433, - "15": 10.83999, - "16": 10.83442, - "17": 10.84483, - "18": 10.81981, - "19": 10.84023, - "20": 10.75974, - "21": 10.74842, - "22": 10.73585, - "23": 10.72846, - "24": 10.70204, - "25": 10.69386, - "26": 10.68294, - "27": 10.65042, - "28": 10.58861, - "29": 10.55276, - "30": 10.52991, - "31": 10.52067, - "32": 10.51487, - "33": 10.47672, - "34": 10.45098, - "35": 10.45204, - "36": 10.42467, - "37": 10.3935, - "38": 10.39796, - "39": 10.36701, - "40": 10.35717, - "41": 10.33082, - "42": 10.31015, - "43": 10.28972, - "44": 10.26722, - "45": 10.27276, - "46": 10.23956, - "47": 10.22509, + "13": 10.86616, + "14": 10.86431, + "15": 10.84011, + "16": 10.83443, + "17": 10.84496, + "18": 10.81997, + "19": 10.84015, + "20": 10.75975, + "21": 10.74843, + "22": 10.73591, + "23": 10.72848, + "24": 10.70201, + "25": 10.69395, + "26": 10.68291, + "27": 10.65044, + "28": 10.58865, + "29": 10.5528, + "30": 10.53001, + "31": 10.52071, + "32": 10.51493, + "33": 10.47669, + "34": 10.45106, + "35": 10.45202, + "36": 10.42475, + "37": 10.39359, + "38": 10.39804, + "39": 10.36707, + "40": 10.35722, + "41": 10.33091, + "42": 10.31022, + "43": 10.28973, + "44": 10.26728, + "45": 10.27285, + "46": 10.23962, + "47": 10.22516, "48": 10.18113, - "49": 10.18221, - "50": 10.18109, - "51": 10.18054, - "52": 10.13848, - "53": 10.14381, - "54": 10.10815, - "55": 10.08196, - "56": 10.10878, - "57": 10.09685, + "49": 10.18226, + "50": 10.1811, + "51": 10.18062, + "52": 10.13852, + "53": 10.14386, + "54": 10.10818, + "55": 10.08201, + "56": 10.10885, + "57": 10.09687, "58": 10.11263, - "59": 10.06067, + "59": 10.06068, "60": 10.08081, - "61": 10.029, - "62": 10.00208, - "63": 10.07351, - "64": 10.02973, - "65": 10.00767, - "66": 10.02968, - "67": 10.00724, - "68": 9.96811, - "69": 9.99247, - "70": 9.96962, - "71": 9.99959, - "72": 9.97932, + "61": 10.02902, + "62": 10.00206, + "63": 10.07353, + "64": 10.02981, + "65": 10.0077, + "66": 10.02965, + "67": 10.00722, + "68": 9.96815, + "69": 9.99249, + "70": 9.96967, + "71": 9.99962, + "72": 9.97935, "73": 9.97019, - "74": 9.9547, - "75": 9.93058, - "76": 9.96382, - "77": 9.95341, - "78": 9.90812, - "79": 9.91503, - "80": 9.92521, - "81": 9.95289, - "82": 9.88577, - "83": 9.85328, - "84": 9.79572, - "85": 9.78589, - "86": 9.87801, - "87": 9.90219, - "88": 9.87507, - "89": 9.82262, - "90": 9.81115, - "91": 9.82642, - "92": 9.81572, - "93": 9.75371, - "94": 9.82278, - "95": 9.81996, - "96": 9.80283, - "97": 9.74847, - "98": 9.77372, - "99": 9.81917, - "100": 9.71201 + "74": 9.95472, + "75": 9.93064, + "76": 9.96386, + "77": 9.95355, + "78": 9.90822, + "79": 9.91513, + "80": 9.92528, + "81": 9.95294, + "82": 9.88587, + "83": 9.85333, + "84": 9.79576, + "85": 9.78595, + "86": 9.87805, + "87": 9.90221, + "88": 9.87509, + "89": 9.82267, + "90": 9.81121, + "91": 9.82646, + "92": 9.81581, + "93": 9.75374, + "94": 9.82283, + "95": 9.81998, + "96": 9.80288, + "97": 9.74858, + "98": 9.77376, + "99": 9.81918, + "100": 9.71205 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1922.0, - "2": 1807.0, - "3": 1882.0, - "4": 1775.0, - "5": 1912.0, - "6": 1714.0, - "7": 1974.0, - "8": 1935.0, - "9": 1833.0, - "10": 1932.0, - "11": 1923.0, - "12": 1769.0, - "13": 1796.0, - "14": 2001.0, - "15": 1767.0, - "16": 1851.0, - "17": 1834.0, - "18": 1796.0, - "19": 1749.0, - "20": 1655.0, - "21": 1824.0, - "22": 1766.0, - "23": 1820.0, - "24": 1931.0, - "25": 1746.0, - "26": 1895.0, - "27": 1859.0, - "28": 1860.0, - "29": 1939.0, - "30": 1870.0, - "31": 2013.0, - "32": 1982.0, - "33": 2040.0, - "34": 2102.0, + "1": 1834.0, + "2": 1741.0, + "3": 1887.0, + "4": 1793.0, + "5": 1911.0, + "6": 1830.0, + "7": 1956.0, + "8": 1826.0, + "9": 1873.0, + "10": 1824.0, + "11": 1830.0, + "12": 1861.0, + "13": 1810.0, + "14": 1928.0, + "15": 1787.0, + "16": 1797.0, + "17": 1926.0, + "18": 1868.0, + "19": 1734.0, + "20": 1707.0, + "21": 2002.0, + "22": 1769.0, + "23": 1784.0, + "24": 1993.0, + "25": 1663.0, + "26": 1799.0, + "27": 1819.0, + "28": 1793.0, + "29": 1978.0, + "30": 1846.0, + "31": 2065.0, + "32": 1998.0, + "33": 1969.0, + "34": 2030.0, "35": 2075.0, - "36": 2026.0, - "37": 2314.0, - "38": 2141.0, - "39": 2189.0, - "40": 2255.0, - "41": 2396.0, - "42": 1953.0, - "43": 2389.0, - "44": 2234.0, - "45": 2588.0, - "46": 2583.0, - "47": 2581.0, - "48": 2757.0, - "49": 2847.0, - "50": 2589.0, - "51": 2438.0, - "52": 2832.0, - "53": 2666.0, - "54": 2900.0, - "55": 2729.0, - "56": 2824.0, - "57": 2182.0, - "58": 3842.0, - "59": 2971.0, - "60": 3042.0, - "61": 2689.0, - "62": 3269.0, - "63": 3494.0, - "64": 3553.0, - "65": 2767.0, - "66": 3146.0, - "67": 3952.0, - "68": 3552.0, - "69": 3054.0, - "70": 3377.0, - "71": 3173.0, - "72": 2909.0, - "73": 3460.0, - "74": 3310.0, - "75": 3240.0, - "76": 3197.0, - "77": 3644.0, - "78": 3322.0, - "79": 3270.0, - "80": 3197.0, - "81": 3406.0, - "82": 2921.0, - "83": 2929.0, - "84": 2873.0, - "85": 2722.0, - "86": 3294.0, - "87": 2854.0, - "88": 3109.0, - "89": 3067.0, - "90": 3859.0, - "91": 2975.0, - "92": 3083.0, - "93": 3074.0, - "94": 3298.0, - "95": 3271.0, - "96": 3531.0, - "97": 3566.0, - "98": 3646.0, - "99": 3246.0, - "100": 3560.0 + "36": 2019.0, + "37": 2275.0, + "38": 2039.0, + "39": 2182.0, + "40": 2303.0, + "41": 2367.0, + "42": 2060.0, + "43": 2514.0, + "44": 2224.0, + "45": 2665.0, + "46": 2587.0, + "47": 2608.0, + "48": 2761.0, + "49": 2875.0, + "50": 2566.0, + "51": 2495.0, + "52": 2745.0, + "53": 2716.0, + "54": 2941.0, + "55": 2646.0, + "56": 2783.0, + "57": 2219.0, + "58": 3749.0, + "59": 3080.0, + "60": 2948.0, + "61": 2729.0, + "62": 3297.0, + "63": 3479.0, + "64": 3568.0, + "65": 2790.0, + "66": 3159.0, + "67": 3953.0, + "68": 3534.0, + "69": 3111.0, + "70": 3394.0, + "71": 3076.0, + "72": 3103.0, + "73": 3502.0, + "74": 3458.0, + "75": 3260.0, + "76": 3172.0, + "77": 3647.0, + "78": 3314.0, + "79": 3252.0, + "80": 3179.0, + "81": 3383.0, + "82": 2774.0, + "83": 3012.0, + "84": 2861.0, + "85": 2746.0, + "86": 3245.0, + "87": 2921.0, + "88": 3137.0, + "89": 3101.0, + "90": 3855.0, + "91": 3068.0, + "92": 2903.0, + "93": 3124.0, + "94": 3341.0, + "95": 3345.0, + "96": 3457.0, + "97": 3584.0, + "98": 3644.0, + "99": 3274.0, + "100": 3526.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.21012, - "3": 2.48683, - "4": 2.08758, - "5": 1.84106, - "6": 2.36818, - "7": 1.78167, - "8": 2.56082, - "9": 2.63238, - "10": 2.116, - "11": 2.29128, - "12": 1.9369, - "13": 2.21414, - "14": 2.3711, - "15": 1.78324, - "16": 2.07424, - "17": 2.64812, - "18": 1.913, - "19": 2.2414, - "20": 1.9884, - "21": 2.29313, - "22": 1.78308, - "23": 2.31951, - "24": 1.83828, - "25": 2.42959, - "26": 2.57941, - "27": 1.94076, - "28": 1.96561, - "29": 1.76661, - "30": 1.97064, - "31": 1.77809, - "32": 2.94694, - "33": 1.78611, - "34": 1.71801, - "35": 2.43344, - "36": 2.30883, - "37": 2.07707, - "38": 1.58036, - "39": 2.52887, - "40": 2.02749, - "41": 2.17248, - "42": 2.71556, - "43": 1.90625, - "44": 2.11783, - "45": 2.53232, - "46": 1.93615, - "47": 2.04965, - "48": 2.37923, - "49": 1.86396, - "50": 2.19078, - "51": 1.54546, - "52": 2.6855, - "53": 2.58466, - "54": 2.62603, - "55": 2.33101, - "56": 3.32308, - "57": 1.6526, - "58": 2.42086, - "59": 2.38312, - "60": 2.32069, - "61": 2.52682, - "62": 2.92065, - "63": 1.99964, - "64": 2.3869, - "65": 2.21157, - "66": 2.47398, - "67": 2.56848, - "68": 1.75833, - "69": 2.37531, - "70": 2.11149, - "71": 2.7953, - "72": 2.23818, - "73": 2.20853, - "74": 2.04207, - "75": 2.11843, - "76": 2.23482, - "77": 2.30573, - "78": 2.30962, - "79": 2.65893, - "80": 2.31419, - "81": 2.74547, - "82": 2.70882, - "83": 2.44411, - "84": 2.01994, - "85": 2.37118, - "86": 2.09735, - "87": 2.8617, - "88": 2.61602, - "89": 2.69622, - "90": 2.55519, - "91": 2.49755, - "92": 3.03609, - "93": 2.12677, - "94": 2.65295, - "95": 2.48188, - "96": 2.68478, - "97": 2.28276, - "98": 2.59049, - "99": 3.33845, - "100": 2.52552 + "2": 5.25238, + "3": 2.71253, + "4": 2.21639, + "5": 1.74789, + "6": 2.13086, + "7": 1.70824, + "8": 2.58642, + "9": 2.5604, + "10": 2.12298, + "11": 2.33739, + "12": 1.81904, + "13": 2.22017, + "14": 2.31721, + "15": 1.77159, + "16": 2.13723, + "17": 2.68046, + "18": 1.94608, + "19": 2.20079, + "20": 1.90305, + "21": 2.33439, + "22": 1.87417, + "23": 2.30248, + "24": 1.774, + "25": 2.54729, + "26": 2.65812, + "27": 2.01009, + "28": 2.02981, + "29": 1.66248, + "30": 1.97269, + "31": 1.71795, + "32": 2.98836, + "33": 1.73161, + "34": 1.61438, + "35": 2.46074, + "36": 2.36413, + "37": 1.90605, + "38": 1.55033, + "39": 2.387, + "40": 2.03851, + "41": 2.09971, + "42": 2.74573, + "43": 1.90317, + "44": 2.15202, + "45": 2.53773, + "46": 1.82607, + "47": 2.04128, + "48": 2.34753, + "49": 1.81272, + "50": 2.16192, + "51": 1.45858, + "52": 2.87407, + "53": 2.34972, + "54": 2.47564, + "55": 2.01679, + "56": 2.94636, + "57": 1.50837, + "58": 2.12747, + "59": 1.87748, + "60": 1.97703, + "61": 2.28537, + "62": 2.6306, + "63": 1.69662, + "64": 2.13544, + "65": 1.89563, + "66": 2.10109, + "67": 2.28835, + "68": 1.50831, + "69": 2.06894, + "70": 1.95744, + "71": 2.44981, + "72": 1.88067, + "73": 1.97617, + "74": 1.85852, + "75": 1.9246, + "76": 2.09768, + "77": 2.04254, + "78": 2.21658, + "79": 2.62298, + "80": 2.46013, + "81": 2.66956, + "82": 2.63259, + "83": 2.56269, + "84": 2.21724, + "85": 2.24738, + "86": 2.24651, + "87": 3.12776, + "88": 3.02733, + "89": 2.76005, + "90": 2.5253, + "91": 2.50725, + "92": 2.8167, + "93": 2.08148, + "94": 2.6625, + "95": 2.52362, + "96": 2.46052, + "97": 2.17136, + "98": 2.4369, + "99": 3.06279, + "100": 2.23329 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json index e317e183d08..4be23b0c372 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89706, - "2": 10.88994, - "3": 10.8831, - "4": 10.88988, - "5": 10.88253, - "6": 10.8874, - "7": 10.88758, - "8": 10.88283, - "9": 10.88381, - "10": 10.88124, - "11": 10.87942, - "12": 10.87716, - "13": 10.87392, - "14": 10.86886, - "15": 10.84732, - "16": 10.83848, - "17": 10.83887, - "18": 10.82811, - "19": 10.83477, - "20": 10.76248, - "21": 10.74325, - "22": 10.72764, - "23": 10.72152, - "24": 10.70346, - "25": 10.69674, - "26": 10.68042, - "27": 10.654, - "28": 10.57729, - "29": 10.55345, - "30": 10.52537, - "31": 10.52801, - "32": 10.51618, - "33": 10.47008, - "34": 10.4406, - "35": 10.44039, - "36": 10.42417, - "37": 10.39196, - "38": 10.39172, - "39": 10.35867, - "40": 10.34947, - "41": 10.32488, - "42": 10.30815, - "43": 10.28559, + "1": 10.89695, + "2": 10.88983, + "3": 10.88313, + "4": 10.88984, + "5": 10.88257, + "6": 10.88737, + "7": 10.88753, + "8": 10.88276, + "9": 10.88373, + "10": 10.8813, + "11": 10.87945, + "12": 10.87721, + "13": 10.87391, + "14": 10.86889, + "15": 10.84742, + "16": 10.8385, + "17": 10.83891, + "18": 10.82804, + "19": 10.83469, + "20": 10.76256, + "21": 10.74328, + "22": 10.72762, + "23": 10.72153, + "24": 10.70345, + "25": 10.69669, + "26": 10.68047, + "27": 10.65412, + "28": 10.57731, + "29": 10.55353, + "30": 10.52543, + "31": 10.52806, + "32": 10.51614, + "33": 10.47016, + "34": 10.44068, + "35": 10.44041, + "36": 10.42432, + "37": 10.392, + "38": 10.39181, + "39": 10.35863, + "40": 10.34956, + "41": 10.32492, + "42": 10.30831, + "43": 10.28562, "44": 10.26537, - "45": 10.26838, - "46": 10.23866, - "47": 10.22366, + "45": 10.2684, + "46": 10.2387, + "47": 10.22376, "48": 10.17493, - "49": 10.18427, - "50": 10.17934, - "51": 10.18127, + "49": 10.18431, + "50": 10.17937, + "51": 10.18132, "52": 10.13579, - "53": 10.13436, - "54": 10.1078, - "55": 10.0851, - "56": 10.10727, - "57": 10.10421, - "58": 10.11001, - "59": 10.04654, - "60": 10.07278, - "61": 10.02695, - "62": 10.00112, - "63": 10.06931, - "64": 10.02132, - "65": 9.98311, - "66": 10.02645, - "67": 10.0049, - "68": 9.96999, - "69": 9.98961, - "70": 9.96903, - "71": 9.99459, - "72": 9.98044, - "73": 9.96536, - "74": 9.95995, - "75": 9.9297, - "76": 9.95438, - "77": 9.95044, + "53": 10.13446, + "54": 10.1079, + "55": 10.08516, + "56": 10.10725, + "57": 10.10425, + "58": 10.11006, + "59": 10.04658, + "60": 10.07284, + "61": 10.02703, + "62": 10.00117, + "63": 10.06929, + "64": 10.02141, + "65": 9.98312, + "66": 10.02646, + "67": 10.00489, + "68": 9.97007, + "69": 9.98965, + "70": 9.96905, + "71": 9.99468, + "72": 9.98046, + "73": 9.96537, + "74": 9.95994, + "75": 9.92969, + "76": 9.95444, + "77": 9.95046, "78": 9.90492, "79": 9.90719, - "80": 9.92276, - "81": 9.94802, - "82": 9.88462, - "83": 9.84481, - "84": 9.78362, - "85": 9.77466, - "86": 9.87595, - "87": 9.90368, - "88": 9.87789, - "89": 9.82816, - "90": 9.81308, - "91": 9.82262, - "92": 9.81558, - "93": 9.75135, - "94": 9.82019, - "95": 9.80988, - "96": 9.79633, - "97": 9.7464, - "98": 9.76836, - "99": 9.82002, - "100": 9.70572 + "80": 9.92282, + "81": 9.94809, + "82": 9.88468, + "83": 9.84487, + "84": 9.78366, + "85": 9.77469, + "86": 9.87604, + "87": 9.90369, + "88": 9.87787, + "89": 9.82814, + "90": 9.81311, + "91": 9.82266, + "92": 9.81563, + "93": 9.7514, + "94": 9.82022, + "95": 9.80985, + "96": 9.79637, + "97": 9.74636, + "98": 9.76834, + "99": 9.82001, + "100": 9.70575 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1873.0, - "2": 1703.0, - "3": 1801.0, - "4": 1750.0, - "5": 1905.0, - "6": 1785.0, - "7": 2037.0, - "8": 1770.0, - "9": 1855.0, - "10": 1768.0, - "11": 1853.0, - "12": 1789.0, - "13": 1844.0, - "14": 1998.0, - "15": 1666.0, - "16": 1838.0, - "17": 1864.0, - "18": 1877.0, - "19": 1721.0, - "20": 1768.0, - "21": 1836.0, - "22": 1810.0, - "23": 1805.0, - "24": 1852.0, - "25": 1700.0, - "26": 1866.0, - "27": 1857.0, - "28": 1983.0, - "29": 1960.0, - "30": 1873.0, - "31": 1994.0, - "32": 1968.0, - "33": 2062.0, - "34": 2072.0, - "35": 2136.0, - "36": 2050.0, - "37": 2255.0, - "38": 2187.0, - "39": 2281.0, - "40": 2282.0, - "41": 2432.0, - "42": 2103.0, - "43": 2372.0, - "44": 2275.0, - "45": 2547.0, - "46": 2458.0, - "47": 2531.0, - "48": 2675.0, - "49": 2879.0, - "50": 2643.0, - "51": 2682.0, - "52": 2750.0, - "53": 2650.0, - "54": 2927.0, - "55": 2782.0, - "56": 2814.0, - "57": 2300.0, - "58": 3764.0, - "59": 3015.0, - "60": 2825.0, - "61": 2787.0, - "62": 3072.0, - "63": 3348.0, - "64": 3672.0, - "65": 2758.0, - "66": 3305.0, - "67": 3796.0, - "68": 3510.0, - "69": 3149.0, - "70": 3372.0, - "71": 3109.0, - "72": 3044.0, - "73": 3476.0, - "74": 3314.0, - "75": 3212.0, - "76": 3352.0, - "77": 3845.0, - "78": 3174.0, - "79": 3428.0, - "80": 3290.0, - "81": 3654.0, - "82": 3055.0, - "83": 3112.0, - "84": 2886.0, - "85": 2844.0, - "86": 3160.0, - "87": 2896.0, - "88": 3019.0, - "89": 3058.0, - "90": 3962.0, - "91": 3083.0, - "92": 2849.0, - "93": 3194.0, - "94": 3174.0, - "95": 3180.0, - "96": 3447.0, - "97": 3513.0, - "98": 3294.0, - "99": 3156.0, - "100": 3173.0 + "1": 1891.0, + "2": 1772.0, + "3": 1833.0, + "4": 1776.0, + "5": 1784.0, + "6": 1757.0, + "7": 2022.0, + "8": 1721.0, + "9": 1845.0, + "10": 1772.0, + "11": 1753.0, + "12": 1818.0, + "13": 1835.0, + "14": 2026.0, + "15": 1720.0, + "16": 1817.0, + "17": 1785.0, + "18": 1833.0, + "19": 1752.0, + "20": 1821.0, + "21": 1927.0, + "22": 1860.0, + "23": 1831.0, + "24": 1919.0, + "25": 1836.0, + "26": 1790.0, + "27": 1928.0, + "28": 1950.0, + "29": 1984.0, + "30": 1786.0, + "31": 2116.0, + "32": 2023.0, + "33": 2116.0, + "34": 2014.0, + "35": 2193.0, + "36": 2059.0, + "37": 2253.0, + "38": 2209.0, + "39": 2236.0, + "40": 2332.0, + "41": 2398.0, + "42": 2056.0, + "43": 2522.0, + "44": 2278.0, + "45": 2596.0, + "46": 2487.0, + "47": 2677.0, + "48": 2718.0, + "49": 2900.0, + "50": 2634.0, + "51": 2567.0, + "52": 2804.0, + "53": 2792.0, + "54": 2945.0, + "55": 2757.0, + "56": 2764.0, + "57": 2211.0, + "58": 3602.0, + "59": 3054.0, + "60": 2947.0, + "61": 2765.0, + "62": 3114.0, + "63": 3345.0, + "64": 3654.0, + "65": 2747.0, + "66": 3186.0, + "67": 3872.0, + "68": 3559.0, + "69": 3093.0, + "70": 3290.0, + "71": 3191.0, + "72": 3101.0, + "73": 3496.0, + "74": 3306.0, + "75": 3337.0, + "76": 3347.0, + "77": 3844.0, + "78": 3209.0, + "79": 3308.0, + "80": 3272.0, + "81": 3617.0, + "82": 2967.0, + "83": 3186.0, + "84": 2884.0, + "85": 2773.0, + "86": 3289.0, + "87": 2849.0, + "88": 3017.0, + "89": 3186.0, + "90": 3781.0, + "91": 3089.0, + "92": 2931.0, + "93": 3300.0, + "94": 3308.0, + "95": 3303.0, + "96": 3432.0, + "97": 3467.0, + "98": 3331.0, + "99": 3041.0, + "100": 3055.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.96215, - "3": 0.40447, - "4": 0.40615, - "5": 0.4052, - "6": 0.3984, - "7": 0.40189, - "8": 0.40565, - "9": 0.4015, - "10": 0.39982, - "11": 0.40336, - "12": 0.40808, - "13": 0.40287, - "14": 0.40086, - "15": 0.40962, - "16": 0.40693, - "17": 0.39911, - "18": 0.40497, - "19": 0.39981, - "20": 0.39833, - "21": 0.40002, - "22": 0.40553, - "23": 0.39778, - "24": 0.3997, - "25": 0.40293, - "26": 0.39987, - "27": 0.39787, - "28": 0.39995, - "29": 0.40733, - "30": 0.40184, - "31": 0.40281, - "32": 0.4007, - "33": 0.40099, - "34": 0.39968, - "35": 0.40118, - "36": 0.40042, - "37": 0.3973, - "38": 0.39656, - "39": 0.39681, - "40": 0.3995, - "41": 0.39914, - "42": 0.39822, - "43": 0.39987, - "44": 0.39748, - "45": 0.39883, - "46": 0.4004, - "47": 0.41492, - "48": 0.4007, - "49": 0.40707, - "50": 0.39954, - "51": 0.53848, - "52": 0.43866, - "53": 0.39439, - "54": 0.39643, - "55": 0.40161, - "56": 0.39595, - "57": 0.40415, - "58": 0.39487, - "59": 0.39625, - "60": 0.4016, - "61": 0.39878, - "62": 0.40098, - "63": 0.39832, - "64": 0.40379, - "65": 0.39826, - "66": 0.40045, - "67": 0.39726, - "68": 0.39694, - "69": 0.39541, - "70": 0.39651, - "71": 0.40066, - "72": 0.39547, - "73": 0.40045, - "74": 0.39279, - "75": 0.39432, - "76": 0.40002, - "77": 0.39686, - "78": 0.40338, - "79": 0.39855, - "80": 0.39739, - "81": 0.39514, - "82": 0.39807, - "83": 0.39541, - "84": 0.40009, - "85": 0.39672, - "86": 0.409, - "87": 0.3964, - "88": 0.40293, - "89": 0.40117, - "90": 0.40761, - "91": 0.40043, - "92": 0.40507, - "93": 0.40323, - "94": 0.40287, - "95": 0.40399, - "96": 0.39454, - "97": 0.3944, - "98": 0.39944, - "99": 0.39466, - "100": 0.39438 + "2": 4.08474, + "3": 0.41564, + "4": 0.42237, + "5": 0.41696, + "6": 0.41098, + "7": 0.41556, + "8": 0.4148, + "9": 0.41562, + "10": 0.40957, + "11": 0.41057, + "12": 0.41329, + "13": 0.40951, + "14": 0.41695, + "15": 0.42029, + "16": 0.42117, + "17": 0.41798, + "18": 0.42014, + "19": 0.41803, + "20": 0.43606, + "21": 0.43479, + "22": 0.42884, + "23": 0.45536, + "24": 0.42239, + "25": 0.43365, + "26": 0.42514, + "27": 0.41263, + "28": 0.41932, + "29": 0.41582, + "30": 0.42344, + "31": 0.42203, + "32": 0.42973, + "33": 0.41998, + "34": 0.42688, + "35": 0.42063, + "36": 0.43004, + "37": 0.42807, + "38": 0.41388, + "39": 0.42848, + "40": 0.41593, + "41": 0.41839, + "42": 0.40765, + "43": 0.40889, + "44": 0.40509, + "45": 0.40447, + "46": 0.40968, + "47": 0.41179, + "48": 0.40837, + "49": 0.4128, + "50": 0.40552, + "51": 0.69321, + "52": 0.45225, + "53": 0.40711, + "54": 0.41035, + "55": 0.41203, + "56": 0.40898, + "57": 0.40701, + "58": 0.40716, + "59": 0.4111, + "60": 0.40736, + "61": 0.41084, + "62": 0.40874, + "63": 0.41184, + "64": 0.40756, + "65": 0.40394, + "66": 0.40598, + "67": 0.40778, + "68": 0.41925, + "69": 0.41313, + "70": 0.40765, + "71": 0.41177, + "72": 0.40621, + "73": 0.41759, + "74": 0.40822, + "75": 0.41448, + "76": 0.41757, + "77": 0.40839, + "78": 0.41189, + "79": 0.4114, + "80": 0.4116, + "81": 0.40977, + "82": 0.41296, + "83": 0.41571, + "84": 0.40929, + "85": 0.41351, + "86": 0.40717, + "87": 0.4091, + "88": 0.41307, + "89": 0.40957, + "90": 0.40829, + "91": 0.41237, + "92": 0.4135, + "93": 0.40835, + "94": 0.40954, + "95": 0.41681, + "96": 0.41045, + "97": 0.40838, + "98": 0.40591, + "99": 0.41068, + "100": 0.41102 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json index 63039c03925..31e6f47dcbc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.89534, - "2": 10.88313, - "3": 10.88058, - "4": 10.8822, - "5": 10.88321, - "6": 10.87484, - "7": 10.87757, - "8": 10.88301, - "9": 10.88054, - "10": 10.87887, - "11": 10.87541, - "12": 10.86979, - "13": 10.86712, - "14": 10.86521, - "15": 10.84124, - "16": 10.83552, - "17": 10.84586, - "18": 10.82014, - "19": 10.84135, + "1": 10.89522, + "2": 10.88315, + "3": 10.88059, + "4": 10.88226, + "5": 10.88328, + "6": 10.87479, + "7": 10.87755, + "8": 10.883, + "9": 10.88055, + "10": 10.87891, + "11": 10.87538, + "12": 10.86983, + "13": 10.86708, + "14": 10.86523, + "15": 10.8413, + "16": 10.83556, + "17": 10.84593, + "18": 10.82029, + "19": 10.84133, "20": 10.76033, "21": 10.7515, - "22": 10.73896, - "23": 10.72982, - "24": 10.70361, - "25": 10.69585, - "26": 10.68595, - "27": 10.65365, - "28": 10.592, - "29": 10.55593, - "30": 10.53219, - "31": 10.52302, - "32": 10.51797, - "33": 10.47935, - "34": 10.45411, - "35": 10.454, - "36": 10.42698, - "37": 10.39699, - "38": 10.40023, - "39": 10.36898, - "40": 10.35931, - "41": 10.33342, - "42": 10.31186, - "43": 10.29213, - "44": 10.26934, - "45": 10.27395, - "46": 10.24145, - "47": 10.227, - "48": 10.18305, - "49": 10.18406, - "50": 10.18231, - "51": 10.18171, - "52": 10.14054, - "53": 10.14639, - "54": 10.11019, - "55": 10.08362, - "56": 10.11023, - "57": 10.09879, - "58": 10.11422, - "59": 10.06263, - "60": 10.08229, - "61": 10.03027, - "62": 10.00397, - "63": 10.07492, - "64": 10.03142, - "65": 10.00976, - "66": 10.03138, - "67": 10.00902, - "68": 9.97028, - "69": 9.99473, - "70": 9.97175, - "71": 10.00142, - "72": 9.98221, - "73": 9.97407, - "74": 9.95811, - "75": 9.93459, - "76": 9.96651, - "77": 9.9566, - "78": 9.91141, - "79": 9.91914, - "80": 9.9304, - "81": 9.95666, - "82": 9.88909, - "83": 9.85746, - "84": 9.79774, - "85": 9.78932, - "86": 9.87974, - "87": 9.90388, - "88": 9.87781, - "89": 9.82415, - "90": 9.81292, - "91": 9.8278, - "92": 9.81697, - "93": 9.75605, - "94": 9.82456, - "95": 9.8217, - "96": 9.80484, - "97": 9.75029, - "98": 9.77497, - "99": 9.82037, - "100": 9.7128 + "22": 10.73902, + "23": 10.7299, + "24": 10.70356, + "25": 10.6959, + "26": 10.68597, + "27": 10.65363, + "28": 10.59201, + "29": 10.55599, + "30": 10.53236, + "31": 10.52309, + "32": 10.51801, + "33": 10.47929, + "34": 10.45416, + "35": 10.45393, + "36": 10.42704, + "37": 10.39711, + "38": 10.40027, + "39": 10.36904, + "40": 10.35941, + "41": 10.33345, + "42": 10.31197, + "43": 10.29218, + "44": 10.26941, + "45": 10.27403, + "46": 10.24156, + "47": 10.22712, + "48": 10.18309, + "49": 10.18407, + "50": 10.18236, + "51": 10.18179, + "52": 10.14057, + "53": 10.14642, + "54": 10.11018, + "55": 10.08366, + "56": 10.11032, + "57": 10.09881, + "58": 10.11426, + "59": 10.06266, + "60": 10.08236, + "61": 10.03029, + "62": 10.00394, + "63": 10.07497, + "64": 10.03145, + "65": 10.00975, + "66": 10.03141, + "67": 10.00898, + "68": 9.97035, + "69": 9.99478, + "70": 9.9718, + "71": 10.00151, + "72": 9.98226, + "73": 9.97409, + "74": 9.95817, + "75": 9.93464, + "76": 9.96655, + "77": 9.95664, + "78": 9.91146, + "79": 9.91925, + "80": 9.93044, + "81": 9.95669, + "82": 9.88913, + "83": 9.8575, + "84": 9.79778, + "85": 9.78935, + "86": 9.8798, + "87": 9.90392, + "88": 9.87786, + "89": 9.82422, + "90": 9.81295, + "91": 9.82784, + "92": 9.81705, + "93": 9.75609, + "94": 9.82461, + "95": 9.82176, + "96": 9.80487, + "97": 9.75041, + "98": 9.77503, + "99": 9.82042, + "100": 9.71285 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1879.0, - "2": 1827.0, - "3": 1946.0, - "4": 1786.0, - "5": 1934.0, - "6": 1687.0, - "7": 1969.0, - "8": 1907.0, - "9": 1922.0, - "10": 1838.0, - "11": 1809.0, - "12": 1821.0, - "13": 1859.0, - "14": 2036.0, - "15": 1774.0, - "16": 1853.0, - "17": 1932.0, - "18": 1854.0, - "19": 1797.0, - "20": 1814.0, - "21": 1858.0, - "22": 1779.0, - "23": 1827.0, - "24": 1837.0, - "25": 1757.0, - "26": 1871.0, - "27": 1867.0, - "28": 1949.0, - "29": 2044.0, - "30": 1926.0, - "31": 2023.0, - "32": 1969.0, - "33": 2007.0, - "34": 2111.0, - "35": 2177.0, - "36": 2119.0, - "37": 2240.0, - "38": 2178.0, - "39": 2220.0, - "40": 2368.0, - "41": 2364.0, - "42": 2047.0, - "43": 2445.0, - "44": 2287.0, - "45": 2599.0, - "46": 2556.0, - "47": 2488.0, - "48": 2719.0, - "49": 2822.0, - "50": 2595.0, - "51": 2602.0, - "52": 2804.0, - "53": 2786.0, - "54": 3027.0, - "55": 2718.0, - "56": 2783.0, - "57": 2218.0, - "58": 3910.0, - "59": 3107.0, - "60": 3134.0, - "61": 2834.0, - "62": 3274.0, - "63": 3534.0, - "64": 3542.0, - "65": 2765.0, - "66": 3175.0, - "67": 4116.0, - "68": 3516.0, - "69": 3088.0, - "70": 3237.0, - "71": 3162.0, - "72": 3061.0, - "73": 3470.0, - "74": 3406.0, - "75": 3253.0, - "76": 3291.0, - "77": 3771.0, - "78": 3318.0, - "79": 3240.0, - "80": 3152.0, - "81": 3521.0, - "82": 2878.0, - "83": 3065.0, - "84": 2924.0, - "85": 2831.0, - "86": 3152.0, - "87": 3004.0, - "88": 3149.0, - "89": 3162.0, - "90": 3938.0, - "91": 2951.0, - "92": 2982.0, - "93": 3147.0, - "94": 3175.0, - "95": 3284.0, - "96": 3419.0, - "97": 3522.0, - "98": 3569.0, - "99": 3208.0, - "100": 3476.0 + "1": 1854.0, + "2": 1858.0, + "3": 1941.0, + "4": 1732.0, + "5": 1920.0, + "6": 1844.0, + "7": 2008.0, + "8": 1823.0, + "9": 1885.0, + "10": 1800.0, + "11": 1804.0, + "12": 1833.0, + "13": 1890.0, + "14": 2028.0, + "15": 1803.0, + "16": 1866.0, + "17": 1906.0, + "18": 1839.0, + "19": 1814.0, + "20": 1782.0, + "21": 1916.0, + "22": 1728.0, + "23": 1825.0, + "24": 1933.0, + "25": 1798.0, + "26": 1881.0, + "27": 1897.0, + "28": 1965.0, + "29": 2001.0, + "30": 1824.0, + "31": 2049.0, + "32": 2032.0, + "33": 2033.0, + "34": 2158.0, + "35": 2254.0, + "36": 2053.0, + "37": 2258.0, + "38": 2168.0, + "39": 2280.0, + "40": 2294.0, + "41": 2441.0, + "42": 2024.0, + "43": 2524.0, + "44": 2292.0, + "45": 2714.0, + "46": 2572.0, + "47": 2539.0, + "48": 2637.0, + "49": 2847.0, + "50": 2702.0, + "51": 2607.0, + "52": 2808.0, + "53": 2784.0, + "54": 2930.0, + "55": 2769.0, + "56": 2776.0, + "57": 2284.0, + "58": 3904.0, + "59": 3072.0, + "60": 3099.0, + "61": 2752.0, + "62": 3299.0, + "63": 3450.0, + "64": 3634.0, + "65": 2732.0, + "66": 3266.0, + "67": 3987.0, + "68": 3531.0, + "69": 3176.0, + "70": 3288.0, + "71": 3101.0, + "72": 3013.0, + "73": 3410.0, + "74": 3499.0, + "75": 3242.0, + "76": 3230.0, + "77": 3823.0, + "78": 3356.0, + "79": 3356.0, + "80": 3297.0, + "81": 3544.0, + "82": 2859.0, + "83": 3021.0, + "84": 2948.0, + "85": 2836.0, + "86": 3273.0, + "87": 3039.0, + "88": 3176.0, + "89": 3121.0, + "90": 3813.0, + "91": 2815.0, + "92": 3053.0, + "93": 3093.0, + "94": 3149.0, + "95": 3211.0, + "96": 3548.0, + "97": 3562.0, + "98": 3617.0, + "99": 3132.0, + "100": 3502.0 } }, "mem-allocated-bytes": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.08928, - "3": 2.4142, - "4": 2.40177, - "5": 1.88805, - "6": 2.50569, - "7": 1.94968, - "8": 2.7794, - "9": 2.94828, - "10": 2.50798, - "11": 3.11261, - "12": 2.80493, - "13": 2.73852, - "14": 3.24157, - "15": 2.46907, - "16": 3.11593, - "17": 3.19334, - "18": 2.31346, - "19": 2.65344, - "20": 2.29061, - "21": 2.72492, - "22": 1.98624, - "23": 2.47979, - "24": 2.1021, - "25": 2.95103, - "26": 3.02023, - "27": 2.29962, - "28": 2.49912, - "29": 2.11753, - "30": 2.24013, - "31": 2.15314, - "32": 4.25568, - "33": 2.69117, - "34": 3.0567, - "35": 3.68884, - "36": 3.19579, - "37": 2.48692, - "38": 2.53056, - "39": 3.09628, - "40": 2.4614, - "41": 2.51819, - "42": 3.1751, - "43": 2.28582, - "44": 2.47582, - "45": 2.82828, - "46": 2.2188, - "47": 2.38142, - "48": 2.77833, - "49": 1.88475, - "50": 2.2008, - "51": 1.57454, - "52": 2.43769, - "53": 2.63045, - "54": 2.19334, - "55": 2.50425, - "56": 3.24638, - "57": 1.69979, - "58": 2.66862, - "59": 2.11892, - "60": 2.45527, - "61": 2.526, - "62": 2.92742, - "63": 2.05047, - "64": 2.40871, - "65": 2.37633, - "66": 2.68518, - "67": 2.59126, - "68": 1.8339, - "69": 2.6468, - "70": 2.2852, - "71": 2.88407, - "72": 2.23295, - "73": 2.2724, - "74": 1.9465, - "75": 1.89306, - "76": 2.1995, - "77": 2.26172, - "78": 2.20307, - "79": 2.62622, - "80": 2.56017, - "81": 2.86228, - "82": 2.92284, - "83": 2.56572, - "84": 2.74261, - "85": 2.88594, - "86": 2.20958, - "87": 3.25945, - "88": 2.60166, - "89": 2.6639, - "90": 3.45798, - "91": 3.13275, - "92": 3.26408, - "93": 2.15378, - "94": 2.76238, - "95": 2.43012, - "96": 2.50507, - "97": 2.23095, - "98": 2.50054, - "99": 3.59538, - "100": 2.39324 + "2": 5.15741, + "3": 2.77281, + "4": 2.28033, + "5": 1.86206, + "6": 2.32966, + "7": 1.80158, + "8": 2.50188, + "9": 2.66695, + "10": 2.22327, + "11": 2.27413, + "12": 2.0255, + "13": 2.13275, + "14": 2.39659, + "15": 1.89571, + "16": 1.93984, + "17": 2.64602, + "18": 1.90043, + "19": 2.19168, + "20": 1.88136, + "21": 2.37005, + "22": 1.78034, + "23": 2.3459, + "24": 1.70595, + "25": 2.52114, + "26": 2.52708, + "27": 1.93244, + "28": 2.04154, + "29": 1.76515, + "30": 1.85421, + "31": 1.773, + "32": 3.00078, + "33": 1.7554, + "34": 1.62127, + "35": 2.71439, + "36": 2.16449, + "37": 2.07304, + "38": 1.58052, + "39": 2.4656, + "40": 1.97982, + "41": 2.04753, + "42": 2.61172, + "43": 1.84615, + "44": 2.00979, + "45": 2.52657, + "46": 1.80379, + "47": 2.06663, + "48": 2.26649, + "49": 1.78687, + "50": 2.07491, + "51": 1.32267, + "52": 2.21054, + "53": 2.30715, + "54": 2.42606, + "55": 2.15456, + "56": 2.83891, + "57": 1.54217, + "58": 2.31097, + "59": 1.80271, + "60": 1.94727, + "61": 2.20492, + "62": 2.50969, + "63": 1.78078, + "64": 2.17352, + "65": 2.18297, + "66": 2.42788, + "67": 2.4889, + "68": 1.67737, + "69": 2.33204, + "70": 2.13254, + "71": 2.63159, + "72": 2.08858, + "73": 2.04183, + "74": 1.85738, + "75": 1.79251, + "76": 2.08909, + "77": 1.8611, + "78": 1.91593, + "79": 2.31911, + "80": 2.13987, + "81": 2.42193, + "82": 2.26799, + "83": 2.1451, + "84": 1.89038, + "85": 1.96471, + "86": 2.17544, + "87": 2.83405, + "88": 2.68051, + "89": 2.51947, + "90": 2.51874, + "91": 2.94024, + "92": 3.16562, + "93": 2.17359, + "94": 2.69828, + "95": 2.34015, + "96": 2.49526, + "97": 2.2401, + "98": 2.51556, + "99": 3.24396, + "100": 2.17511 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json index 2fac2b44381..34abdb88474 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 544347136.0, - "2": 544348672.0, - "3": 544348672.0, - "4": 544348672.0, - "5": 544348672.0, - "6": 544348672.0, - "7": 544348672.0, - "8": 544348672.0, - "9": 544348672.0, - "10": 544348672.0, - "11": 544348672.0, - "12": 544348672.0, - "13": 544348672.0, - "14": 544348672.0, - "15": 544348672.0, - "16": 544348672.0, - "17": 544348672.0, - "18": 544348672.0, - "19": 544348672.0, - "20": 544348672.0, - "21": 544348672.0, - "22": 544348672.0, - "23": 544348672.0, - "24": 544348672.0, - "25": 544348672.0, - "26": 544348672.0, - "27": 544348672.0, - "28": 544348672.0, - "29": 544348672.0, - "30": 544348672.0, - "31": 544348672.0, - "32": 544348672.0, - "33": 544348672.0, - "34": 544348672.0, - "35": 544348672.0, - "36": 544348672.0, - "37": 544348672.0, - "38": 544348672.0, - "39": 544348672.0, - "40": 544348672.0, - "41": 544348672.0, - "42": 544348672.0, - "43": 544348672.0, - "44": 544348672.0, - "45": 544348672.0, - "46": 544348672.0, - "47": 544348672.0, - "48": 544348672.0, - "49": 544348672.0, - "50": 544348672.0 + "1": 594416640.0, + "2": 594418176.0, + "3": 594418176.0, + "4": 594418176.0, + "5": 594418176.0, + "6": 594418176.0, + "7": 594418176.0, + "8": 594418176.0, + "9": 594418176.0, + "10": 594418176.0, + "11": 594418176.0, + "12": 594418176.0, + "13": 594418176.0, + "14": 594418176.0, + "15": 594418176.0, + "16": 594418176.0, + "17": 594418176.0, + "18": 594418176.0, + "19": 594418176.0, + "20": 594418176.0, + "21": 594418176.0, + "22": 594418176.0, + "23": 594418176.0, + "24": 594418176.0, + "25": 594418176.0, + "26": 594418176.0, + "27": 594418176.0, + "28": 594418176.0, + "29": 594418176.0, + "30": 594418176.0, + "31": 594418176.0, + "32": 594418176.0, + "33": 594418176.0, + "34": 594418176.0, + "35": 594418176.0, + "36": 594418176.0, + "37": 594418176.0, + "38": 594418176.0, + "39": 594418176.0, + "40": 594418176.0, + "41": 594418176.0, + "42": 594418176.0, + "43": 594418176.0, + "44": 594418176.0, + "45": 594418176.0, + "46": 594418176.0, + "47": 594418176.0, + "48": 594418176.0, + "49": 594418176.0, + "50": 594418176.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2335718400.0, - "2": 2458891776.0, - "3": 2479774208.0, - "4": 2479774208.0, - "5": 2479774208.0, - "6": 2479774208.0, - "7": 2479774208.0, - "8": 2479774208.0, - "9": 2479774208.0, - "10": 2479774208.0, - "11": 2479774208.0, - "12": 2479774208.0, - "13": 2479774208.0, - "14": 2479774208.0, - "15": 2479774208.0, - "16": 2479774208.0, - "17": 2479774208.0, - "18": 2479774208.0, - "19": 2479774208.0, - "20": 2479774208.0, - "21": 2479774208.0, - "22": 2479774208.0, - "23": 2479774208.0, - "24": 2479774208.0, - "25": 2479774208.0, - "26": 2479774208.0, - "27": 2479774208.0, - "28": 2479774208.0, - "29": 2479774208.0, - "30": 2479774208.0, - "31": 2479774208.0, - "32": 2479774208.0, - "33": 2479774208.0, - "34": 2479774208.0, - "35": 2479774208.0, - "36": 2479774208.0, - "37": 2479774208.0, - "38": 2479774208.0, - "39": 2481645568.0, - "40": 2481645568.0, - "41": 2481645568.0, - "42": 2481645568.0, - "43": 2481645568.0, - "44": 2481645568.0, - "45": 2489437184.0, - "46": 2489437184.0, - "47": 2489437184.0, - "48": 2489437184.0, - "49": 2489437184.0, - "50": 2489437184.0 + "1": 2385787904.0, + "2": 2508961280.0, + "3": 2529843712.0, + "4": 2529843712.0, + "5": 2529843712.0, + "6": 2529843712.0, + "7": 2529843712.0, + "8": 2529843712.0, + "9": 2529843712.0, + "10": 2529843712.0, + "11": 2529843712.0, + "12": 2529843712.0, + "13": 2529843712.0, + "14": 2529843712.0, + "15": 2529843712.0, + "16": 2529843712.0, + "17": 2529843712.0, + "18": 2529843712.0, + "19": 2529843712.0, + "20": 2529843712.0, + "21": 2529843712.0, + "22": 2529843712.0, + "23": 2529843712.0, + "24": 2529843712.0, + "25": 2529843712.0, + "26": 2529843712.0, + "27": 2529843712.0, + "28": 2529843712.0, + "29": 2529843712.0, + "30": 2529843712.0, + "31": 2529843712.0, + "32": 2529843712.0, + "33": 2529843712.0, + "34": 2529843712.0, + "35": 2529843712.0, + "36": 2529843712.0, + "37": 2529843712.0, + "38": 2529843712.0, + "39": 2531715072.0, + "40": 2531715072.0, + "41": 2531715072.0, + "42": 2531715072.0, + "43": 2531715072.0, + "44": 2531715072.0, + "45": 2539506688.0, + "46": 2539506688.0, + "47": 2539506688.0, + "48": 2539506688.0, + "49": 2539506688.0, + "50": 2539506688.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 13.12071, - "3": 0.14634, - "4": 0.55256, - "5": 0.74839, - "6": 0.76356, - "7": 0.38394, - "8": 0.50723, - "9": 0.93992, - "10": 0.50174, - "11": 0.67907, - "12": 0.69163, - "13": 0.48596, - "14": 0.52873, - "15": 0.44691, - "16": 0.537, - "17": 0.59548, - "18": 0.8704, - "19": 1.26369, - "20": 0.36655, - "21": 0.39997, - "22": 0.54097, - "23": 1.28904, - "24": 0.72709, - "25": 0.23533, - "26": 0.59206, - "27": 1.13639, - "28": 0.44584, - "29": 0.53564, - "30": 0.70672, - "31": 0.91962, - "32": 0.35207, - "33": 1.0285, - "34": 0.70842, - "35": 0.5221, - "36": 0.99081, - "37": 0.63678, - "38": 0.30022, - "39": 0.69391, - "40": 0.79304, - "41": 0.68078, - "42": 0.56356, - "43": 0.51748, - "44": 0.81076, - "45": 0.98012, - "46": 0.41141, - "47": 0.70899, - "48": 0.68674, - "49": 0.56065, - "50": 1.00066 + "2": 12.78616, + "3": 0.1263, + "4": 0.69944, + "5": 0.70311, + "6": 0.72094, + "7": 0.33756, + "8": 0.55932, + "9": 0.73242, + "10": 0.57684, + "11": 0.90862, + "12": 1.02775, + "13": 0.68326, + "14": 0.51122, + "15": 0.4671, + "16": 0.59291, + "17": 0.65569, + "18": 0.58508, + "19": 1.01652, + "20": 0.54363, + "21": 0.39484, + "22": 0.90435, + "23": 0.80357, + "24": 0.77363, + "25": 0.43172, + "26": 0.77249, + "27": 0.96313, + "28": 0.47261, + "29": 0.51333, + "30": 0.81788, + "31": 0.60892, + "32": 0.4855, + "33": 0.97321, + "34": 0.64729, + "35": 0.46514, + "36": 0.84552, + "37": 0.46938, + "38": 0.52313, + "39": 0.43514, + "40": 0.5703, + "41": 0.91146, + "42": 0.51738, + "43": 0.39267, + "44": 0.46113, + "45": 1.04646, + "46": 0.46625, + "47": 0.74725, + "48": 0.92515, + "49": 0.46575, + "50": 1.09022 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json index d4eaa62dbaf..66a948b1666 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92624, - "2": 10.91761, - "3": 10.92333, - "4": 10.92688, - "5": 10.92998, - "6": 10.92516, + "1": 10.92561, + "2": 10.91856, + "3": 10.92257, + "4": 10.92726, + "5": 10.92964, + "6": 10.92362, "7": 10.9275, - "8": 10.91969, - "9": 10.92241, - "10": 10.92325, - "11": 10.9076, - "12": 10.92305, - "13": 10.9061, - "14": 10.88901, - "15": 10.89569, - "16": 10.87458, - "17": 10.88061, - "18": 10.87989, - "19": 10.87749, - "20": 10.82246, - "21": 10.82186, - "22": 10.7974, - "23": 10.80849, - "24": 10.77057, - "25": 10.77862, - "26": 10.75668, - "27": 10.74862, - "28": 10.68541, - "29": 10.65859, - "30": 10.62605, - "31": 10.61208, - "32": 10.60629, - "33": 10.57851, - "34": 10.53548, + "8": 10.91971, + "9": 10.92189, + "10": 10.92381, + "11": 10.90772, + "12": 10.92237, + "13": 10.907, + "14": 10.88924, + "15": 10.89606, + "16": 10.87509, + "17": 10.88069, + "18": 10.87988, + "19": 10.8779, + "20": 10.82287, + "21": 10.82214, + "22": 10.79726, + "23": 10.80932, + "24": 10.77051, + "25": 10.77867, + "26": 10.75636, + "27": 10.74899, + "28": 10.68475, + "29": 10.65911, + "30": 10.6272, + "31": 10.61296, + "32": 10.6061, + "33": 10.57894, + "34": 10.536, "35": 10.53752, - "36": 10.52815, - "37": 10.49657, - "38": 10.49494, - "39": 10.46031, - "40": 10.44224, - "41": 10.42194, - "42": 10.41512, - "43": 10.39777, - "44": 10.36628, - "45": 10.37246, - "46": 10.33638, - "47": 10.31798, - "48": 10.28671, - "49": 10.27533, - "50": 10.27489, - "51": 10.27015, - "52": 10.22169, - "53": 10.22939, - "54": 10.19442, - "55": 10.17402, - "56": 10.18836, - "57": 10.17883, - "58": 10.18533, - "59": 10.13453, - "60": 10.15581, - "61": 10.10616, - "62": 10.074, - "63": 10.13216, - "64": 10.09238, - "65": 10.06787, - "66": 10.09506, - "67": 10.07264, - "68": 10.03606, - "69": 10.05538, - "70": 10.03315, - "71": 10.04934, - "72": 10.04115, - "73": 10.03896, - "74": 10.0216, + "36": 10.52821, + "37": 10.4971, + "38": 10.49446, + "39": 10.46075, + "40": 10.44199, + "41": 10.42128, + "42": 10.41473, + "43": 10.39834, + "44": 10.36684, + "45": 10.37233, + "46": 10.33579, + "47": 10.31804, + "48": 10.28691, + "49": 10.27523, + "50": 10.27494, + "51": 10.26955, + "52": 10.22127, + "53": 10.22941, + "54": 10.19422, + "55": 10.17429, + "56": 10.18853, + "57": 10.17897, + "58": 10.18526, + "59": 10.13483, + "60": 10.15585, + "61": 10.10615, + "62": 10.07405, + "63": 10.13177, + "64": 10.09204, + "65": 10.06791, + "66": 10.095, + "67": 10.07294, + "68": 10.03607, + "69": 10.05491, + "70": 10.03321, + "71": 10.04987, + "72": 10.04091, + "73": 10.03878, + "74": 10.02114, "75": 9.9823, - "76": 10.00321, - "77": 10.0147, - "78": 9.96885, - "79": 9.96967, - "80": 9.98518, - "81": 10.01118, - "82": 9.94219, - "83": 9.91692, - "84": 9.8533, - "85": 9.83713, - "86": 9.94372, - "87": 9.95587, - "88": 9.93524, - "89": 9.87174, - "90": 9.87886, - "91": 9.89547, - "92": 9.86943, - "93": 9.80146, - "94": 9.88892, - "95": 9.86048, - "96": 9.84941, - "97": 9.79913, - "98": 9.83243, - "99": 9.8647, - "100": 9.75978 + "76": 10.00354, + "77": 10.01483, + "78": 9.96817, + "79": 9.96957, + "80": 9.98504, + "81": 10.01108, + "82": 9.9421, + "83": 9.91677, + "84": 9.85306, + "85": 9.83728, + "86": 9.94391, + "87": 9.95597, + "88": 9.93542, + "89": 9.87129, + "90": 9.87899, + "91": 9.89527, + "92": 9.86926, + "93": 9.80185, + "94": 9.88937, + "95": 9.86027, + "96": 9.84922, + "97": 9.79903, + "98": 9.83241, + "99": 9.86401, + "100": 9.76018 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36362.0, - "2": 36609.0, - "3": 36699.0, - "4": 36555.0, - "5": 36453.0, - "6": 36089.0, - "7": 37058.0, - "8": 36230.0, - "9": 37158.0, - "10": 37488.0, - "11": 36351.0, - "12": 35467.0, - "13": 36613.0, - "14": 36715.0, - "15": 36545.0, - "16": 35601.0, - "17": 36415.0, - "18": 36855.0, - "19": 36832.0, - "20": 36630.0, - "21": 35929.0, - "22": 36615.0, - "23": 36863.0, - "24": 36540.0, - "25": 36141.0, - "26": 36330.0, - "27": 36011.0, - "28": 35481.0, - "29": 35864.0, - "30": 35285.0, - "31": 36267.0, - "32": 36439.0, - "33": 35284.0, - "34": 35815.0, - "35": 36651.0, - "36": 35025.0, - "37": 36022.0, - "38": 35730.0, - "39": 36407.0, - "40": 37543.0, - "41": 36402.0, - "42": 35346.0, - "43": 37029.0, - "44": 36347.0, - "45": 39301.0, - "46": 38134.0, - "47": 38278.0, - "48": 38458.0, - "49": 42002.0, - "50": 38854.0, - "51": 38657.0, - "52": 41903.0, - "53": 40194.0, - "54": 41670.0, - "55": 38786.0, - "56": 40937.0, - "57": 36754.0, - "58": 45577.0, - "59": 42100.0, - "60": 42166.0, - "61": 41783.0, - "62": 44364.0, - "63": 43830.0, - "64": 50186.0, - "65": 41853.0, - "66": 45150.0, - "67": 50276.0, - "68": 47034.0, - "69": 45199.0, - "70": 47305.0, - "71": 45654.0, - "72": 44705.0, - "73": 47912.0, - "74": 45912.0, - "75": 47295.0, - "76": 46874.0, - "77": 47833.0, - "78": 49256.0, - "79": 47375.0, - "80": 45364.0, - "81": 53599.0, - "82": 42559.0, - "83": 46334.0, - "84": 45573.0, - "85": 45415.0, - "86": 45108.0, - "87": 36933.0, - "88": 46121.0, - "89": 49063.0, - "90": 56728.0, - "91": 38149.0, - "92": 49957.0, - "93": 46429.0, - "94": 43030.0, - "95": 45818.0, - "96": 50189.0, - "97": 47382.0, - "98": 49426.0, - "99": 43744.0, - "100": 45484.0 + "1": 36436.0, + "2": 36501.0, + "3": 36802.0, + "4": 36766.0, + "5": 36386.0, + "6": 35733.0, + "7": 37148.0, + "8": 36168.0, + "9": 37700.0, + "10": 37692.0, + "11": 36380.0, + "12": 35362.0, + "13": 36444.0, + "14": 37198.0, + "15": 36113.0, + "16": 36080.0, + "17": 35890.0, + "18": 36677.0, + "19": 36875.0, + "20": 36654.0, + "21": 35934.0, + "22": 36630.0, + "23": 36555.0, + "24": 36447.0, + "25": 35789.0, + "26": 36432.0, + "27": 36480.0, + "28": 35574.0, + "29": 35892.0, + "30": 35261.0, + "31": 36004.0, + "32": 36774.0, + "33": 35429.0, + "34": 35421.0, + "35": 36348.0, + "36": 35172.0, + "37": 36118.0, + "38": 35314.0, + "39": 36245.0, + "40": 37579.0, + "41": 35853.0, + "42": 35642.0, + "43": 36534.0, + "44": 36152.0, + "45": 39483.0, + "46": 38103.0, + "47": 37913.0, + "48": 38654.0, + "49": 42357.0, + "50": 38920.0, + "51": 38654.0, + "52": 41728.0, + "53": 40126.0, + "54": 41345.0, + "55": 38968.0, + "56": 40444.0, + "57": 36849.0, + "58": 45895.0, + "59": 41886.0, + "60": 42200.0, + "61": 41553.0, + "62": 44568.0, + "63": 43769.0, + "64": 49912.0, + "65": 41452.0, + "66": 45193.0, + "67": 49905.0, + "68": 47291.0, + "69": 45132.0, + "70": 47912.0, + "71": 45345.0, + "72": 45252.0, + "73": 47856.0, + "74": 46077.0, + "75": 46976.0, + "76": 47228.0, + "77": 47619.0, + "78": 48947.0, + "79": 47363.0, + "80": 45590.0, + "81": 53548.0, + "82": 42539.0, + "83": 46174.0, + "84": 46192.0, + "85": 45427.0, + "86": 45044.0, + "87": 37111.0, + "88": 46323.0, + "89": 49563.0, + "90": 56927.0, + "91": 38107.0, + "92": 49565.0, + "93": 46649.0, + "94": 42544.0, + "95": 45963.0, + "96": 49752.0, + "97": 47239.0, + "98": 49652.0, + "99": 44234.0, + "100": 45362.0 } }, "mem-allocated-bytes": { @@ -218,49 +218,49 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 892871168.0, - "2": 892866048.0, - "3": 892867584.0, - "4": 892867584.0, + "1": 892870144.0, + "2": 892867584.0, + "3": 892866560.0, + "4": 892868096.0, "5": 892868096.0, - "6": 892871168.0, - "7": 892869632.0, - "8": 892867072.0, - "9": 892868096.0, + "6": 892871680.0, + "7": 892870144.0, + "8": 892866560.0, + "9": 892867072.0, "10": 892866560.0, - "11": 892868096.0, - "12": 892865536.0, - "13": 892869632.0, - "14": 892869632.0, - "15": 892868096.0, - "16": 892866560.0, - "17": 892867584.0, - "18": 892866560.0, - "19": 892872704.0, - "20": 892866048.0, - "21": 892867072.0, - "22": 892868096.0, + "11": 892869120.0, + "12": 892867072.0, + "13": 892869120.0, + "14": 892867584.0, + "15": 892868608.0, + "16": 892867072.0, + "17": 892866560.0, + "18": 892867584.0, + "19": 892871168.0, + "20": 892867072.0, + "21": 892866560.0, + "22": 892867584.0, "23": 892868608.0, "24": 892868096.0, - "25": 892866560.0, + "25": 892868096.0, "26": 892864512.0, - "27": 892867584.0, - "28": 892867584.0, - "29": 892871168.0, - "30": 892867072.0, - "31": 892868096.0, - "32": 892865536.0, - "33": 892868096.0, - "34": 892866048.0, - "35": 892866560.0, - "36": 892868608.0, - "37": 892869632.0, - "38": 892866560.0, + "27": 892869120.0, + "28": 892868608.0, + "29": 892869632.0, + "30": 892866560.0, + "31": 892867072.0, + "32": 892866048.0, + "33": 892868608.0, + "34": 892867584.0, + "35": 892866048.0, + "36": 892868096.0, + "37": 892870144.0, + "38": 892867072.0, "39": 892867584.0, - "40": 892869632.0, - "41": 892869120.0, - "42": 892867072.0, - "43": 892865536.0, + "40": 892870656.0, + "41": 892868096.0, + "42": 892868096.0, + "43": 892864512.0, "44": 892867584.0, "45": 892868096.0, "46": 892868608.0, @@ -268,56 +268,56 @@ "48": 892866048.0, "49": 892866048.0, "50": 892869632.0, - "51": 892865024.0, - "52": 892868096.0, + "51": 892866048.0, + "52": 892867072.0, "53": 892866560.0, - "54": 892865536.0, - "55": 892868608.0, - "56": 892866048.0, - "57": 892868096.0, - "58": 892868608.0, - "59": 892866560.0, - "60": 892865536.0, - "61": 892865024.0, + "54": 892867072.0, + "55": 892869120.0, + "56": 892867584.0, + "57": 892868608.0, + "58": 892868096.0, + "59": 892867072.0, + "60": 892865024.0, + "61": 892865536.0, "62": 892866048.0, - "63": 892865024.0, - "64": 892867584.0, - "65": 892866048.0, - "66": 892868096.0, - "67": 892865536.0, - "68": 892868608.0, + "63": 892865536.0, + "64": 892868096.0, + "65": 892867072.0, + "66": 892870144.0, + "67": 892866048.0, + "68": 892869120.0, "69": 892866048.0, - "70": 892864512.0, - "71": 892866560.0, + "70": 892866048.0, + "71": 892867584.0, "72": 892870144.0, - "73": 892866048.0, - "74": 892866048.0, + "73": 892865536.0, + "74": 892865024.0, "75": 892868608.0, - "76": 892866560.0, - "77": 892861440.0, - "78": 892866560.0, - "79": 892867072.0, + "76": 892868096.0, + "77": 892862464.0, + "78": 892864512.0, + "79": 892865536.0, "80": 892865024.0, "81": 892861952.0, - "82": 892862464.0, + "82": 892862976.0, "83": 892867072.0, - "84": 892861440.0, - "85": 892859392.0, - "86": 892853248.0, - "87": 892865536.0, - "88": 892869120.0, - "89": 892857856.0, - "90": 892859904.0, - "91": 892866560.0, - "92": 892855808.0, - "93": 892861440.0, + "84": 892863488.0, + "85": 892859904.0, + "86": 892852736.0, + "87": 892865024.0, + "88": 892868096.0, + "89": 892858368.0, + "90": 892859392.0, + "91": 892866048.0, + "92": 892854272.0, + "93": 892861952.0, "94": 892855808.0, "95": 892853248.0, - "96": 892867584.0, - "97": 892853760.0, - "98": 892847104.0, - "99": 892852736.0, - "100": 892852224.0 + "96": 892868096.0, + "97": 892853248.0, + "98": 892846592.0, + "99": 892853760.0, + "100": 892851712.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1921131008.0, - "2": 2156760064.0, - "3": 2156760064.0, - "4": 2156760064.0, - "5": 2156760064.0, - "6": 2159165952.0, - "7": 2159165952.0, - "8": 2159165952.0, - "9": 2159165952.0, - "10": 2159165952.0, - "11": 2159165952.0, - "12": 2159165952.0, - "13": 2159165952.0, - "14": 2159165952.0, - "15": 2159165952.0, - "16": 2159165952.0, - "17": 2159165952.0, - "18": 2159165952.0, - "19": 2159417344.0, - "20": 2159417344.0, - "21": 2159417344.0, - "22": 2159417344.0, - "23": 2159417344.0, - "24": 2159417344.0, - "25": 2159417344.0, - "26": 2159417344.0, - "27": 2159417344.0, - "28": 2159417344.0, - "29": 2159417344.0, - "30": 2159417344.0, - "31": 2159417344.0, - "32": 2159417344.0, - "33": 2159417344.0, - "34": 2159417344.0, - "35": 2159417344.0, - "36": 2159417344.0, - "37": 2159417344.0, - "38": 2159417344.0, - "39": 2159417344.0, - "40": 2159417344.0, - "41": 2159417344.0, - "42": 2159417344.0, - "43": 2159417344.0, - "44": 2159417344.0, - "45": 2159417344.0, - "46": 2159417344.0, - "47": 2159417344.0, - "48": 2159417344.0, - "49": 2159417344.0, - "50": 2159417344.0, - "51": 2159417344.0, - "52": 2159417344.0, - "53": 2159417344.0, - "54": 2159417344.0, - "55": 2159417344.0, - "56": 2159417344.0, - "57": 2159417344.0, - "58": 2159417344.0, - "59": 2159417344.0, - "60": 2159417344.0, - "61": 2159417344.0, - "62": 2159417344.0, - "63": 2159417344.0, - "64": 2159417344.0, - "65": 2159417344.0, - "66": 2159417344.0, - "67": 2159417344.0, - "68": 2159417344.0, - "69": 2159417344.0, - "70": 2159417344.0, - "71": 2159417344.0, - "72": 2159417344.0, - "73": 2159417344.0, - "74": 2159417344.0, - "75": 2159417344.0, - "76": 2159417344.0, - "77": 2159417344.0, - "78": 2159417344.0, - "79": 2159417344.0, - "80": 2159417344.0, - "81": 2159417344.0, - "82": 2159417344.0, - "83": 2159417344.0, - "84": 2159417344.0, - "85": 2159417344.0, - "86": 2159417344.0, - "87": 2159417344.0, - "88": 2159417344.0, - "89": 2159417344.0, - "90": 2159417344.0, - "91": 2159417344.0, - "92": 2159417344.0, - "93": 2159417344.0, - "94": 2159417344.0, - "95": 2159417344.0, - "96": 2159417344.0, - "97": 2159417344.0, - "98": 2159417344.0, - "99": 2159417344.0, - "100": 2159417344.0 + "1": 1921322496.0, + "2": 2156503040.0, + "3": 2156503040.0, + "4": 2156758528.0, + "5": 2156758528.0, + "6": 2159042560.0, + "7": 2159042560.0, + "8": 2159042560.0, + "9": 2159042560.0, + "10": 2159042560.0, + "11": 2159042560.0, + "12": 2159042560.0, + "13": 2159042560.0, + "14": 2159042560.0, + "15": 2159042560.0, + "16": 2159042560.0, + "17": 2159042560.0, + "18": 2159042560.0, + "19": 2159231488.0, + "20": 2159231488.0, + "21": 2159231488.0, + "22": 2159231488.0, + "23": 2159231488.0, + "24": 2159231488.0, + "25": 2159231488.0, + "26": 2159231488.0, + "27": 2159231488.0, + "28": 2159231488.0, + "29": 2159231488.0, + "30": 2159231488.0, + "31": 2159231488.0, + "32": 2159231488.0, + "33": 2159231488.0, + "34": 2159231488.0, + "35": 2159231488.0, + "36": 2159231488.0, + "37": 2159231488.0, + "38": 2159231488.0, + "39": 2159231488.0, + "40": 2159231488.0, + "41": 2159231488.0, + "42": 2159231488.0, + "43": 2159231488.0, + "44": 2159231488.0, + "45": 2159231488.0, + "46": 2159231488.0, + "47": 2159231488.0, + "48": 2159231488.0, + "49": 2159231488.0, + "50": 2159231488.0, + "51": 2159231488.0, + "52": 2159231488.0, + "53": 2159231488.0, + "54": 2159231488.0, + "55": 2159231488.0, + "56": 2159231488.0, + "57": 2159231488.0, + "58": 2159231488.0, + "59": 2159231488.0, + "60": 2159231488.0, + "61": 2159231488.0, + "62": 2159231488.0, + "63": 2159231488.0, + "64": 2159231488.0, + "65": 2159231488.0, + "66": 2159231488.0, + "67": 2159231488.0, + "68": 2159231488.0, + "69": 2159231488.0, + "70": 2159231488.0, + "71": 2159231488.0, + "72": 2159231488.0, + "73": 2159231488.0, + "74": 2159231488.0, + "75": 2159231488.0, + "76": 2159231488.0, + "77": 2159231488.0, + "78": 2159231488.0, + "79": 2159231488.0, + "80": 2159231488.0, + "81": 2159231488.0, + "82": 2159231488.0, + "83": 2159231488.0, + "84": 2159231488.0, + "85": 2159231488.0, + "86": 2159231488.0, + "87": 2159231488.0, + "88": 2159231488.0, + "89": 2159231488.0, + "90": 2159231488.0, + "91": 2159231488.0, + "92": 2159231488.0, + "93": 2159231488.0, + "94": 2159231488.0, + "95": 2159231488.0, + "96": 2159231488.0, + "97": 2159231488.0, + "98": 2159231488.0, + "99": 2159231488.0, + "100": 2159231488.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.16725, - "3": 0.32945, - "4": 0.3326, - "5": 0.31532, - "6": 0.31313, - "7": 0.317, - "8": 0.30642, - "9": 0.30646, - "10": 0.3022, - "11": 0.30703, - "12": 0.30536, - "13": 0.30606, - "14": 0.30467, - "15": 0.3118, - "16": 0.30695, - "17": 0.30446, - "18": 0.30389, - "19": 0.30558, - "20": 0.30186, - "21": 0.30526, - "22": 0.3017, - "23": 0.30526, - "24": 0.30353, - "25": 0.30816, - "26": 0.30525, - "27": 0.30507, - "28": 0.30162, - "29": 0.29981, - "30": 0.30343, - "31": 0.30716, - "32": 0.29956, - "33": 0.29728, - "34": 0.29985, - "35": 0.29717, - "36": 0.30103, - "37": 0.30189, - "38": 0.30242, - "39": 0.30086, - "40": 0.3021, - "41": 0.30098, - "42": 0.29986, - "43": 0.30059, - "44": 0.30081, - "45": 0.29867, - "46": 0.30811, - "47": 0.3014, - "48": 0.3011, - "49": 0.30087, - "50": 0.3016, - "51": 0.36223, - "52": 2.63153, - "53": 0.30957, - "54": 0.30865, - "55": 0.30761, - "56": 0.30931, - "57": 0.30991, - "58": 0.31171, - "59": 0.30828, - "60": 0.30579, - "61": 0.30747, - "62": 0.31163, - "63": 0.3087, - "64": 0.31316, - "65": 0.30963, - "66": 0.3059, - "67": 0.30317, - "68": 0.30933, - "69": 0.30347, - "70": 0.30489, - "71": 0.31189, - "72": 0.31243, - "73": 0.3091, - "74": 0.31819, - "75": 0.33007, - "76": 0.3104, - "77": 0.31692, - "78": 0.31918, - "79": 0.32085, - "80": 0.32266, - "81": 0.33123, - "82": 0.31385, - "83": 0.36905, - "84": 0.33251, - "85": 0.33023, - "86": 0.33809, - "87": 0.30414, - "88": 0.31558, - "89": 0.3397, - "90": 0.32143, - "91": 0.33501, - "92": 0.32793, - "93": 0.34383, - "94": 0.33469, - "95": 0.3472, - "96": 0.37909, - "97": 0.34492, - "98": 0.34483, - "99": 0.33437, - "100": 0.34951 + "2": 7.72368, + "3": 0.34461, + "4": 0.3435, + "5": 0.3341, + "6": 0.3277, + "7": 0.33352, + "8": 0.3281, + "9": 0.32749, + "10": 0.32881, + "11": 0.32254, + "12": 0.33968, + "13": 0.33108, + "14": 0.33481, + "15": 0.33178, + "16": 0.32753, + "17": 0.32842, + "18": 0.3236, + "19": 0.32469, + "20": 0.32504, + "21": 0.32591, + "22": 0.31857, + "23": 0.33399, + "24": 0.32926, + "25": 0.32259, + "26": 0.33177, + "27": 0.32836, + "28": 0.32202, + "29": 0.32057, + "30": 0.32982, + "31": 0.32391, + "32": 0.32983, + "33": 0.31796, + "34": 0.32727, + "35": 0.32317, + "36": 0.32169, + "37": 0.31852, + "38": 0.33164, + "39": 0.32446, + "40": 0.32075, + "41": 0.3222, + "42": 0.31832, + "43": 0.32426, + "44": 0.32022, + "45": 0.31987, + "46": 0.31832, + "47": 0.32396, + "48": 0.31946, + "49": 0.31729, + "50": 0.32087, + "51": 0.38744, + "52": 2.6474, + "53": 0.31848, + "54": 0.32049, + "55": 0.31497, + "56": 0.32124, + "57": 0.3172, + "58": 0.3211, + "59": 0.32147, + "60": 0.32528, + "61": 0.33032, + "62": 0.3166, + "63": 0.32186, + "64": 0.31794, + "65": 0.31628, + "66": 0.31467, + "67": 0.32065, + "68": 0.35005, + "69": 0.35693, + "70": 0.35687, + "71": 0.35269, + "72": 0.37218, + "73": 0.34516, + "74": 0.35899, + "75": 0.38136, + "76": 0.36329, + "77": 0.37986, + "78": 0.37573, + "79": 0.36922, + "80": 0.3623, + "81": 0.33636, + "82": 0.32253, + "83": 0.38119, + "84": 0.36072, + "85": 0.33747, + "86": 0.34586, + "87": 0.3177, + "88": 0.32841, + "89": 0.36741, + "90": 0.34372, + "91": 0.34948, + "92": 0.33962, + "93": 0.3451, + "94": 0.3523, + "95": 0.36669, + "96": 0.38572, + "97": 0.36505, + "98": 0.36259, + "99": 0.33937, + "100": 0.36424 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 6dbbfe4def7..b267dfdabd4 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.91066, - "2": 10.89646, - "3": 10.90792, - "4": 10.90307, - "5": 10.91195, - "6": 10.90646, - "7": 10.89838, - "8": 10.90122, - "9": 10.90026, - "10": 10.90331, - "11": 10.89166, - "12": 10.89988, - "13": 10.8849, - "14": 10.87998, - "15": 10.8728, - "16": 10.86504, - "17": 10.86869, - "18": 10.8571, - "19": 10.86119, - "20": 10.8188, - "21": 10.80335, - "22": 10.78966, - "23": 10.77604, - "24": 10.74885, - "25": 10.75645, - "26": 10.74017, - "27": 10.72496, - "28": 10.67041, - "29": 10.63542, - "30": 10.61272, - "31": 10.60548, - "32": 10.58806, - "33": 10.56518, - "34": 10.53433, - "35": 10.53075, - "36": 10.51437, - "37": 10.47627, - "38": 10.48915, - "39": 10.44952, - "40": 10.43444, - "41": 10.41455, - "42": 10.40199, - "43": 10.38762, - "44": 10.34257, - "45": 10.36188, - "46": 10.32335, - "47": 10.3067, - "48": 10.27172, - "49": 10.26262, - "50": 10.2613, - "51": 10.25781, - "52": 10.21493, - "53": 10.22583, - "54": 10.17883, - "55": 10.16505, - "56": 10.17871, - "57": 10.1692, - "58": 10.17718, - "59": 10.1302, - "60": 10.14386, - "61": 10.09356, - "62": 10.07265, - "63": 10.12749, - "64": 10.09284, - "65": 10.07125, - "66": 10.08262, - "67": 10.06175, - "68": 10.02605, - "69": 10.04908, - "70": 10.02759, - "71": 10.04607, - "72": 10.03034, - "73": 10.02624, - "74": 10.00701, - "75": 9.98506, - "76": 10.00662, - "77": 10.00431, - "78": 9.95602, - "79": 9.96789, - "80": 9.98225, - "81": 10.01113, - "82": 9.94293, - "83": 9.92061, - "84": 9.84992, - "85": 9.84827, - "86": 9.9353, - "87": 9.96845, - "88": 9.93373, - "89": 9.87283, - "90": 9.87687, - "91": 9.89287, - "92": 9.875, - "93": 9.81107, - "94": 9.89072, - "95": 9.87368, - "96": 9.85278, - "97": 9.7965, - "98": 9.83453, - "99": 9.87169, - "100": 9.76389 + "1": 10.91078, + "2": 10.89671, + "3": 10.9068, + "4": 10.90309, + "5": 10.91231, + "6": 10.90619, + "7": 10.89729, + "8": 10.90225, + "9": 10.89974, + "10": 10.90271, + "11": 10.89174, + "12": 10.89972, + "13": 10.88445, + "14": 10.88017, + "15": 10.8721, + "16": 10.86401, + "17": 10.86921, + "18": 10.85741, + "19": 10.8598, + "20": 10.81848, + "21": 10.80338, + "22": 10.78816, + "23": 10.77549, + "24": 10.74977, + "25": 10.75623, + "26": 10.74088, + "27": 10.72474, + "28": 10.6708, + "29": 10.63634, + "30": 10.61286, + "31": 10.60565, + "32": 10.58807, + "33": 10.56535, + "34": 10.53443, + "35": 10.53086, + "36": 10.5136, + "37": 10.47664, + "38": 10.48988, + "39": 10.44993, + "40": 10.43435, + "41": 10.41501, + "42": 10.40213, + "43": 10.38757, + "44": 10.34276, + "45": 10.36178, + "46": 10.32389, + "47": 10.30661, + "48": 10.27197, + "49": 10.26302, + "50": 10.26171, + "51": 10.25787, + "52": 10.2153, + "53": 10.22533, + "54": 10.17887, + "55": 10.16493, + "56": 10.17821, + "57": 10.1693, + "58": 10.17732, + "59": 10.13049, + "60": 10.14409, + "61": 10.09377, + "62": 10.07236, + "63": 10.1274, + "64": 10.09298, + "65": 10.07117, + "66": 10.0824, + "67": 10.06131, + "68": 10.02633, + "69": 10.04916, + "70": 10.02744, + "71": 10.04579, + "72": 10.03071, + "73": 10.02627, + "74": 10.00693, + "75": 9.9848, + "76": 10.00692, + "77": 10.00432, + "78": 9.95648, + "79": 9.96782, + "80": 9.98203, + "81": 10.01091, + "82": 9.94328, + "83": 9.92101, + "84": 9.84991, + "85": 9.84859, + "86": 9.93522, + "87": 9.96824, + "88": 9.93385, + "89": 9.87306, + "90": 9.87695, + "91": 9.89293, + "92": 9.87527, + "93": 9.81105, + "94": 9.8906, + "95": 9.87358, + "96": 9.85291, + "97": 9.79664, + "98": 9.83443, + "99": 9.87124, + "100": 9.76382 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 37001.0, - "2": 36732.0, - "3": 36500.0, - "4": 36497.0, - "5": 36308.0, - "6": 35804.0, - "7": 37128.0, - "8": 35982.0, - "9": 36339.0, - "10": 37420.0, - "11": 36825.0, - "12": 35918.0, - "13": 36509.0, - "14": 36718.0, - "15": 36144.0, - "16": 35879.0, - "17": 36557.0, - "18": 36406.0, - "19": 36689.0, - "20": 36332.0, - "21": 36165.0, - "22": 36186.0, - "23": 36662.0, - "24": 36628.0, - "25": 35283.0, - "26": 36188.0, - "27": 36438.0, - "28": 35557.0, - "29": 35655.0, - "30": 35347.0, - "31": 36045.0, - "32": 36682.0, - "33": 36030.0, - "34": 35613.0, - "35": 36767.0, - "36": 35693.0, - "37": 36422.0, - "38": 35167.0, - "39": 36773.0, - "40": 37229.0, - "41": 36203.0, - "42": 35749.0, - "43": 37587.0, - "44": 36237.0, - "45": 39444.0, - "46": 37852.0, - "47": 38394.0, - "48": 38946.0, - "49": 42609.0, - "50": 39507.0, - "51": 39090.0, - "52": 41821.0, - "53": 40567.0, - "54": 41610.0, - "55": 39318.0, - "56": 41334.0, - "57": 36899.0, - "58": 46231.0, - "59": 42246.0, - "60": 42784.0, - "61": 42495.0, - "62": 45631.0, - "63": 44361.0, - "64": 49155.0, - "65": 42460.0, - "66": 45659.0, - "67": 50043.0, - "68": 46333.0, - "69": 45862.0, - "70": 46950.0, - "71": 46684.0, - "72": 45346.0, - "73": 47664.0, - "74": 45501.0, - "75": 48985.0, - "76": 47531.0, - "77": 48431.0, - "78": 48651.0, - "79": 48300.0, - "80": 45717.0, - "81": 54912.0, - "82": 42473.0, - "83": 46906.0, - "84": 45830.0, - "85": 46189.0, - "86": 46183.0, - "87": 36768.0, - "88": 45219.0, - "89": 48522.0, - "90": 58149.0, - "91": 39290.0, - "92": 49092.0, - "93": 45067.0, - "94": 42178.0, - "95": 45548.0, - "96": 50098.0, - "97": 46773.0, - "98": 49620.0, - "99": 46056.0, - "100": 46551.0 + "1": 36671.0, + "2": 37197.0, + "3": 36559.0, + "4": 36852.0, + "5": 36703.0, + "6": 35840.0, + "7": 36983.0, + "8": 36383.0, + "9": 36544.0, + "10": 37416.0, + "11": 36558.0, + "12": 35768.0, + "13": 36071.0, + "14": 36725.0, + "15": 36398.0, + "16": 35673.0, + "17": 36265.0, + "18": 36819.0, + "19": 36835.0, + "20": 36479.0, + "21": 36112.0, + "22": 36404.0, + "23": 36252.0, + "24": 36731.0, + "25": 35963.0, + "26": 36050.0, + "27": 36455.0, + "28": 35396.0, + "29": 35673.0, + "30": 35389.0, + "31": 36231.0, + "32": 36304.0, + "33": 35334.0, + "34": 35822.0, + "35": 36789.0, + "36": 35934.0, + "37": 35999.0, + "38": 36004.0, + "39": 36913.0, + "40": 37742.0, + "41": 35768.0, + "42": 35680.0, + "43": 37957.0, + "44": 36315.0, + "45": 39567.0, + "46": 38316.0, + "47": 38689.0, + "48": 39256.0, + "49": 42420.0, + "50": 39683.0, + "51": 39232.0, + "52": 41791.0, + "53": 40285.0, + "54": 41889.0, + "55": 39269.0, + "56": 41161.0, + "57": 36797.0, + "58": 46655.0, + "59": 42337.0, + "60": 42800.0, + "61": 42084.0, + "62": 45422.0, + "63": 44419.0, + "64": 49741.0, + "65": 42303.0, + "66": 45995.0, + "67": 49650.0, + "68": 46704.0, + "69": 45620.0, + "70": 46489.0, + "71": 46470.0, + "72": 45280.0, + "73": 47307.0, + "74": 45593.0, + "75": 48865.0, + "76": 47482.0, + "77": 48068.0, + "78": 48859.0, + "79": 47823.0, + "80": 45845.0, + "81": 55013.0, + "82": 41851.0, + "83": 46834.0, + "84": 45934.0, + "85": 46257.0, + "86": 45740.0, + "87": 36471.0, + "88": 45175.0, + "89": 48591.0, + "90": 57540.0, + "91": 38958.0, + "92": 49460.0, + "93": 44750.0, + "94": 41945.0, + "95": 45537.0, + "96": 49807.0, + "97": 46836.0, + "98": 49860.0, + "99": 45566.0, + "100": 46454.0 } }, "mem-allocated-bytes": { @@ -218,104 +218,104 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254509056.0, - "2": 1254505984.0, - "3": 1254509056.0, + "1": 1254510592.0, + "2": 1254503936.0, + "3": 1254510080.0, "4": 1254507520.0, "5": 1254503936.0, - "6": 1254505472.0, - "7": 1254505472.0, + "6": 1254504960.0, + "7": 1254507008.0, "8": 1254505984.0, - "9": 1254509056.0, - "10": 1254509056.0, + "9": 1254510592.0, + "10": 1254508032.0, "11": 1254504960.0, "12": 1254507520.0, - "13": 1254508544.0, - "14": 1254507008.0, - "15": 1254505472.0, + "13": 1254508032.0, + "14": 1254506496.0, + "15": 1254507008.0, "16": 1254508032.0, "17": 1254507008.0, - "18": 1254507520.0, - "19": 1254504960.0, - "20": 1254509056.0, - "21": 1254507008.0, - "22": 1254505984.0, + "18": 1254506496.0, + "19": 1254505472.0, + "20": 1254509568.0, + "21": 1254505984.0, + "22": 1254506496.0, "23": 1254505472.0, - "24": 1254503936.0, - "25": 1254507520.0, - "26": 1254508032.0, - "27": 1254508544.0, - "28": 1254507008.0, - "29": 1254508544.0, - "30": 1254508032.0, - "31": 1254508544.0, - "32": 1254504960.0, + "24": 1254503424.0, + "25": 1254508032.0, + "26": 1254506496.0, + "27": 1254508032.0, + "28": 1254506496.0, + "29": 1254508032.0, + "30": 1254508544.0, + "31": 1254506496.0, + "32": 1254504448.0, "33": 1254506496.0, - "34": 1254504960.0, - "35": 1254504960.0, + "34": 1254506496.0, + "35": 1254504448.0, "36": 1254504960.0, - "37": 1254508032.0, - "38": 1254509056.0, - "39": 1254505984.0, - "40": 1254506496.0, - "41": 1254505472.0, + "37": 1254508544.0, + "38": 1254507520.0, + "39": 1254505472.0, + "40": 1254507008.0, + "41": 1254506496.0, "42": 1254504960.0, - "43": 1254507008.0, - "44": 1254503936.0, - "45": 1254504960.0, + "43": 1254506496.0, + "44": 1254504448.0, + "45": 1254504448.0, "46": 1254505984.0, "47": 1254506496.0, - "48": 1254506496.0, - "49": 1254508032.0, + "48": 1254507008.0, + "49": 1254507520.0, "50": 1254504448.0, - "51": 1254501376.0, + "51": 1254503424.0, "52": 1254506496.0, - "53": 1254503424.0, - "54": 1254503424.0, - "55": 1254505472.0, + "53": 1254504448.0, + "54": 1254504448.0, + "55": 1254506496.0, "56": 1254504448.0, "57": 1254507008.0, - "58": 1254508544.0, - "59": 1254505472.0, + "58": 1254507008.0, + "59": 1254504960.0, "60": 1254505984.0, - "61": 1254503424.0, + "61": 1254502912.0, "62": 1254505984.0, - "63": 1254506496.0, - "64": 1254505472.0, - "65": 1254507008.0, + "63": 1254505472.0, + "64": 1254505984.0, + "65": 1254505984.0, "66": 1254507008.0, - "67": 1254504448.0, - "68": 1254505984.0, - "69": 1254504960.0, - "70": 1254505472.0, + "67": 1254503936.0, + "68": 1254504448.0, + "69": 1254505472.0, + "70": 1254506496.0, "71": 1254502400.0, - "72": 1254504448.0, - "73": 1254502912.0, + "72": 1254503936.0, + "73": 1254502400.0, "74": 1254505984.0, "75": 1254505984.0, "76": 1254505472.0, - "77": 1254505984.0, - "78": 1254502912.0, - "79": 1254506496.0, + "77": 1254505472.0, + "78": 1254501376.0, + "79": 1254504960.0, "80": 1254505984.0, - "81": 1254505472.0, + "81": 1254504960.0, "82": 1254505472.0, - "83": 1254503424.0, + "83": 1254504448.0, "84": 1254505984.0, - "85": 1254504960.0, - "86": 1254506496.0, - "87": 1254504448.0, - "88": 1254502400.0, - "89": 1254509568.0, + "85": 1254503424.0, + "86": 1254505984.0, + "87": 1254502912.0, + "88": 1254501376.0, + "89": 1254508032.0, "90": 1254507008.0, - "91": 1254503936.0, + "91": 1254504448.0, "92": 1254505984.0, "93": 1254507008.0, - "94": 1254507520.0, - "95": 1254511104.0, - "96": 1254504448.0, - "97": 1254509568.0, - "98": 1254507520.0, + "94": 1254506496.0, + "95": 1254513664.0, + "96": 1254503936.0, + "97": 1254509056.0, + "98": 1254508032.0, "99": 1254508032.0, "100": 1254509056.0 } @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2065569280.0, - "2": 2546124800.0, - "3": 2546124800.0, - "4": 2546124800.0, - "5": 2546257920.0, - "6": 2546257920.0, - "7": 2546452480.0, - "8": 2546452480.0, - "9": 2546452480.0, - "10": 2546452480.0, - "11": 2546452480.0, - "12": 2546452480.0, - "13": 2546452480.0, - "14": 2546452480.0, - "15": 2546452480.0, - "16": 2546452480.0, - "17": 2546452480.0, - "18": 2546452480.0, - "19": 2546452480.0, - "20": 2546452480.0, - "21": 2546452480.0, - "22": 2546452480.0, - "23": 2546452480.0, - "24": 2546452480.0, - "25": 2546452480.0, - "26": 2546452480.0, - "27": 2546452480.0, - "28": 2546452480.0, - "29": 2546452480.0, - "30": 2546452480.0, - "31": 2546757632.0, - "32": 2546757632.0, - "33": 2546757632.0, - "34": 2546757632.0, - "35": 2546757632.0, - "36": 2546757632.0, - "37": 2546757632.0, - "38": 2546757632.0, - "39": 2546757632.0, - "40": 2546757632.0, - "41": 2546757632.0, - "42": 2546757632.0, - "43": 2546757632.0, - "44": 2546757632.0, - "45": 2546757632.0, - "46": 2546757632.0, - "47": 2546757632.0, - "48": 2546757632.0, - "49": 2546757632.0, - "50": 2546757632.0, - "51": 2546757632.0, - "52": 2546757632.0, - "53": 2546757632.0, - "54": 2546757632.0, - "55": 2546757632.0, - "56": 2546757632.0, - "57": 2546757632.0, - "58": 2546757632.0, - "59": 2546757632.0, - "60": 2546757632.0, - "61": 2546757632.0, - "62": 2546757632.0, - "63": 2546757632.0, - "64": 2546757632.0, - "65": 2546757632.0, - "66": 2546757632.0, - "67": 2546757632.0, - "68": 2546757632.0, - "69": 2546757632.0, - "70": 2546757632.0, - "71": 2546757632.0, - "72": 2546757632.0, - "73": 2546757632.0, - "74": 2546757632.0, - "75": 2546757632.0, - "76": 2546757632.0, - "77": 2546757632.0, - "78": 2546757632.0, - "79": 2546757632.0, - "80": 2546757632.0, - "81": 2546757632.0, - "82": 2546757632.0, - "83": 2546757632.0, - "84": 2546757632.0, - "85": 2546757632.0, - "86": 2546757632.0, - "87": 2546757632.0, - "88": 2546757632.0, - "89": 2546757632.0, - "90": 2546757632.0, - "91": 2546757632.0, - "92": 2546757632.0, - "93": 2548838912.0, - "94": 2548838912.0, - "95": 2549586432.0, - "96": 2549586432.0, - "97": 2549698048.0, - "98": 2549698048.0, - "99": 2549698048.0, - "100": 2553959936.0 + "1": 2065977344.0, + "2": 2545691648.0, + "3": 2546333696.0, + "4": 2546333696.0, + "5": 2546333696.0, + "6": 2546333696.0, + "7": 2546546688.0, + "8": 2546546688.0, + "9": 2546546688.0, + "10": 2546546688.0, + "11": 2546546688.0, + "12": 2546546688.0, + "13": 2546546688.0, + "14": 2546546688.0, + "15": 2546546688.0, + "16": 2546546688.0, + "17": 2546546688.0, + "18": 2546546688.0, + "19": 2546546688.0, + "20": 2546546688.0, + "21": 2546546688.0, + "22": 2546546688.0, + "23": 2546546688.0, + "24": 2546625024.0, + "25": 2546625024.0, + "26": 2546625024.0, + "27": 2546625024.0, + "28": 2546625024.0, + "29": 2546625024.0, + "30": 2546625024.0, + "31": 2546924544.0, + "32": 2546924544.0, + "33": 2546924544.0, + "34": 2546924544.0, + "35": 2546924544.0, + "36": 2546924544.0, + "37": 2546924544.0, + "38": 2546924544.0, + "39": 2546924544.0, + "40": 2546924544.0, + "41": 2546924544.0, + "42": 2546924544.0, + "43": 2546924544.0, + "44": 2546924544.0, + "45": 2546924544.0, + "46": 2546924544.0, + "47": 2546924544.0, + "48": 2546924544.0, + "49": 2546924544.0, + "50": 2546924544.0, + "51": 2546924544.0, + "52": 2546924544.0, + "53": 2546924544.0, + "54": 2546924544.0, + "55": 2546924544.0, + "56": 2546924544.0, + "57": 2546924544.0, + "58": 2546924544.0, + "59": 2546924544.0, + "60": 2546924544.0, + "61": 2546924544.0, + "62": 2546924544.0, + "63": 2546924544.0, + "64": 2546924544.0, + "65": 2546924544.0, + "66": 2546924544.0, + "67": 2546924544.0, + "68": 2546924544.0, + "69": 2546924544.0, + "70": 2546924544.0, + "71": 2546924544.0, + "72": 2546924544.0, + "73": 2546924544.0, + "74": 2546924544.0, + "75": 2546924544.0, + "76": 2546924544.0, + "77": 2546924544.0, + "78": 2546924544.0, + "79": 2546924544.0, + "80": 2546924544.0, + "81": 2546924544.0, + "82": 2546924544.0, + "83": 2546924544.0, + "84": 2546924544.0, + "85": 2548597248.0, + "86": 2548597248.0, + "87": 2548597248.0, + "88": 2548597248.0, + "89": 2548597248.0, + "90": 2548597248.0, + "91": 2548597248.0, + "92": 2548597248.0, + "93": 2550968832.0, + "94": 2550968832.0, + "95": 2550968832.0, + "96": 2550968832.0, + "97": 2551127552.0, + "98": 2551127552.0, + "99": 2551127552.0, + "100": 2552623616.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.35065, - "3": 1.49589, - "4": 1.34581, - "5": 0.80643, - "6": 1.43894, - "7": 1.37891, - "8": 1.49563, - "9": 1.95785, - "10": 0.85481, - "11": 1.33899, - "12": 1.43755, - "13": 1.19509, - "14": 1.83328, - "15": 0.98032, - "16": 1.40874, - "17": 1.66864, - "18": 1.5365, - "19": 1.30779, - "20": 1.79944, - "21": 0.71488, - "22": 1.32647, - "23": 1.24779, - "24": 2.20621, - "25": 1.10326, - "26": 0.96542, - "27": 1.29006, - "28": 1.41795, - "29": 1.89483, - "30": 0.79813, - "31": 0.97715, - "32": 2.08206, - "33": 1.21236, - "34": 0.71896, - "35": 1.46986, - "36": 1.39013, - "37": 1.24449, - "38": 0.73381, - "39": 1.35307, - "40": 1.7201, - "41": 1.25122, - "42": 1.33799, - "43": 1.08317, - "44": 1.41303, - "45": 1.08191, - "46": 0.79454, - "47": 1.23184, - "48": 1.40187, - "49": 1.10435, - "50": 1.36801, - "51": 0.84116, - "52": 1.02029, - "53": 1.39142, - "54": 2.03947, - "55": 1.50361, - "56": 1.31223, - "57": 0.93054, - "58": 1.61309, - "59": 1.19898, - "60": 1.08312, - "61": 1.63845, - "62": 1.43415, - "63": 1.40495, - "64": 1.31806, - "65": 1.06803, - "66": 1.3885, - "67": 1.59941, - "68": 1.09038, - "69": 1.70706, - "70": 1.28046, - "71": 1.41662, - "72": 1.50138, - "73": 1.2025, - "74": 0.91636, - "75": 1.18148, - "76": 1.36065, - "77": 1.41942, - "78": 1.33114, - "79": 1.88785, - "80": 1.08232, - "81": 1.31472, - "82": 1.68157, - "83": 1.15155, - "84": 1.18209, - "85": 0.9033, - "86": 1.09546, - "87": 1.81244, - "88": 1.1479, - "89": 1.00561, - "90": 1.55058, - "91": 0.97663, - "92": 1.58765, - "93": 1.01357, - "94": 1.08504, - "95": 1.09183, - "96": 1.88433, - "97": 1.03535, - "98": 0.77708, - "99": 1.69156, - "100": 1.06124 + "2": 5.16676, + "3": 1.33849, + "4": 1.23335, + "5": 0.73686, + "6": 1.28572, + "7": 1.09349, + "8": 1.3093, + "9": 1.3909, + "10": 0.97654, + "11": 1.23723, + "12": 1.25701, + "13": 1.28286, + "14": 1.73245, + "15": 1.27684, + "16": 1.53975, + "17": 1.44709, + "18": 1.28735, + "19": 1.54823, + "20": 1.23997, + "21": 1.2885, + "22": 1.20702, + "23": 1.5628, + "24": 1.82853, + "25": 0.70485, + "26": 1.41088, + "27": 1.24459, + "28": 1.68268, + "29": 1.09092, + "30": 0.87799, + "31": 1.1021, + "32": 2.05649, + "33": 1.57849, + "34": 0.99956, + "35": 1.70846, + "36": 0.97953, + "37": 1.04274, + "38": 0.76389, + "39": 1.31153, + "40": 1.44917, + "41": 1.15798, + "42": 1.23798, + "43": 1.27748, + "44": 1.53042, + "45": 1.01719, + "46": 1.00468, + "47": 1.28352, + "48": 1.23432, + "49": 1.03827, + "50": 1.43566, + "51": 0.71174, + "52": 1.01744, + "53": 1.40236, + "54": 1.47379, + "55": 1.34089, + "56": 1.23548, + "57": 0.97281, + "58": 1.32399, + "59": 0.92588, + "60": 0.98813, + "61": 1.31305, + "62": 1.31408, + "63": 1.10625, + "64": 1.41175, + "65": 0.83015, + "66": 1.30029, + "67": 1.34831, + "68": 1.04914, + "69": 1.60422, + "70": 0.99852, + "71": 1.30909, + "72": 1.20603, + "73": 1.12393, + "74": 0.74197, + "75": 1.15464, + "76": 1.13503, + "77": 1.3649, + "78": 1.33475, + "79": 1.93519, + "80": 1.06877, + "81": 1.16103, + "82": 1.52244, + "83": 1.22539, + "84": 1.12373, + "85": 0.81898, + "86": 0.99191, + "87": 1.92012, + "88": 1.06987, + "89": 0.98033, + "90": 1.55702, + "91": 1.03794, + "92": 1.76067, + "93": 1.0433, + "94": 1.29033, + "95": 1.17477, + "96": 1.20314, + "97": 1.0363, + "98": 1.44299, + "99": 1.54793, + "100": 1.20692 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json index 025ee773139..85f82b37739 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92624, - "2": 10.91761, - "3": 10.92333, - "4": 10.92688, - "5": 10.92998, - "6": 10.92516, + "1": 10.92561, + "2": 10.91856, + "3": 10.92257, + "4": 10.92726, + "5": 10.92964, + "6": 10.92362, "7": 10.9275, - "8": 10.91969, - "9": 10.92241, - "10": 10.92384, - "11": 10.90735, - "12": 10.92256, - "13": 10.90603, - "14": 10.88893, - "15": 10.89574, - "16": 10.87437, - "17": 10.88111, + "8": 10.9196, + "9": 10.92217, + "10": 10.92392, + "11": 10.90759, + "12": 10.92243, + "13": 10.90721, + "14": 10.88991, + "15": 10.89617, + "16": 10.87581, + "17": 10.87957, "18": 10.87989, - "19": 10.8777, - "20": 10.8229, - "21": 10.82151, - "22": 10.79833, - "23": 10.8093, - "24": 10.76998, - "25": 10.77846, - "26": 10.75625, - "27": 10.74877, - "28": 10.68549, - "29": 10.65912, - "30": 10.62682, - "31": 10.61174, - "32": 10.60594, - "33": 10.57889, - "34": 10.53573, - "35": 10.53734, - "36": 10.52861, - "37": 10.49704, - "38": 10.49542, - "39": 10.46092, - "40": 10.44217, - "41": 10.42129, - "42": 10.41516, - "43": 10.39816, - "44": 10.36695, - "45": 10.37226, - "46": 10.33554, - "47": 10.31809, - "48": 10.28656, - "49": 10.27539, - "50": 10.27508, - "51": 10.26992, - "52": 10.22163, - "53": 10.22906, - "54": 10.19419, - "55": 10.1737, - "56": 10.18831, - "57": 10.17869, - "58": 10.18534, - "59": 10.13498, - "60": 10.15562, - "61": 10.10616, - "62": 10.07447, - "63": 10.13218, - "64": 10.09215, - "65": 10.06779, - "66": 10.09487, - "67": 10.07265, - "68": 10.03581, - "69": 10.05526, - "70": 10.03316, - "71": 10.04956, - "72": 10.04089, - "73": 10.03831, - "74": 10.02118, - "75": 9.98217, - "76": 10.00359, - "77": 10.01464, - "78": 9.96825, - "79": 9.96968, - "80": 9.98497, - "81": 10.01098, - "82": 9.942, - "83": 9.91659, - "84": 9.85291, - "85": 9.83766, - "86": 9.94369, + "19": 10.87757, + "20": 10.82261, + "21": 10.82203, + "22": 10.79825, + "23": 10.80864, + "24": 10.76984, + "25": 10.77858, + "26": 10.756, + "27": 10.7495, + "28": 10.68555, + "29": 10.65925, + "30": 10.62653, + "31": 10.61181, + "32": 10.6054, + "33": 10.57914, + "34": 10.5363, + "35": 10.53794, + "36": 10.52773, + "37": 10.49731, + "38": 10.49549, + "39": 10.46025, + "40": 10.44235, + "41": 10.42131, + "42": 10.41507, + "43": 10.39818, + "44": 10.36665, + "45": 10.37237, + "46": 10.3366, + "47": 10.31802, + "48": 10.28685, + "49": 10.27543, + "50": 10.27528, + "51": 10.26998, + "52": 10.22134, + "53": 10.22978, + "54": 10.19405, + "55": 10.17419, + "56": 10.18856, + "57": 10.17853, + "58": 10.18551, + "59": 10.13513, + "60": 10.15563, + "61": 10.10614, + "62": 10.07409, + "63": 10.13178, + "64": 10.09178, + "65": 10.06805, + "66": 10.09521, + "67": 10.0727, + "68": 10.03552, + "69": 10.05548, + "70": 10.03291, + "71": 10.04966, + "72": 10.04135, + "73": 10.03852, + "74": 10.0208, + "75": 9.98234, + "76": 10.00344, + "77": 10.01482, + "78": 9.96829, + "79": 9.96986, + "80": 9.98503, + "81": 10.01099, + "82": 9.94184, + "83": 9.91679, + "84": 9.85314, + "85": 9.83762, + "86": 9.94372, "87": 9.95599, - "88": 9.93523, - "89": 9.87132, - "90": 9.8789, - "91": 9.89514, - "92": 9.86906, - "93": 9.8012, - "94": 9.8889, - "95": 9.86015, - "96": 9.849, - "97": 9.79912, - "98": 9.83254, - "99": 9.8645, - "100": 9.76002 + "88": 9.93505, + "89": 9.87147, + "90": 9.87922, + "91": 9.89549, + "92": 9.86949, + "93": 9.80192, + "94": 9.88899, + "95": 9.86036, + "96": 9.84943, + "97": 9.79943, + "98": 9.83227, + "99": 9.86459, + "100": 9.76036 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36362.0, - "2": 36609.0, - "3": 36699.0, - "4": 36555.0, - "5": 36453.0, - "6": 36089.0, - "7": 37058.0, - "8": 36230.0, - "9": 37141.0, - "10": 37547.0, - "11": 36627.0, - "12": 35253.0, - "13": 36871.0, - "14": 36920.0, - "15": 36125.0, - "16": 35933.0, - "17": 36498.0, - "18": 36647.0, - "19": 36906.0, - "20": 36544.0, - "21": 36047.0, - "22": 36411.0, - "23": 36227.0, - "24": 36399.0, - "25": 36161.0, - "26": 36511.0, - "27": 36607.0, - "28": 35615.0, - "29": 35641.0, - "30": 35308.0, - "31": 35928.0, - "32": 36626.0, - "33": 35309.0, - "34": 35493.0, - "35": 36313.0, - "36": 35348.0, - "37": 35771.0, - "38": 35727.0, - "39": 36751.0, - "40": 37556.0, - "41": 36096.0, - "42": 35169.0, - "43": 37377.0, - "44": 36413.0, - "45": 39294.0, - "46": 38016.0, - "47": 38125.0, - "48": 38571.0, - "49": 41964.0, - "50": 38513.0, - "51": 38718.0, - "52": 42043.0, - "53": 40110.0, - "54": 41376.0, - "55": 38983.0, - "56": 40743.0, - "57": 37045.0, - "58": 45842.0, - "59": 41774.0, - "60": 42246.0, - "61": 41868.0, - "62": 44805.0, - "63": 43800.0, - "64": 49706.0, - "65": 41660.0, - "66": 45388.0, - "67": 50065.0, - "68": 47387.0, - "69": 44392.0, - "70": 47481.0, - "71": 45682.0, - "72": 45161.0, - "73": 48046.0, - "74": 45712.0, - "75": 47057.0, - "76": 46874.0, - "77": 47480.0, - "78": 49142.0, - "79": 46944.0, - "80": 45550.0, - "81": 54133.0, - "82": 41978.0, - "83": 46216.0, - "84": 45475.0, - "85": 45262.0, - "86": 45607.0, - "87": 37027.0, - "88": 46017.0, - "89": 48872.0, - "90": 56923.0, - "91": 38456.0, - "92": 49398.0, - "93": 46732.0, - "94": 42779.0, - "95": 45860.0, - "96": 50249.0, - "97": 47569.0, - "98": 49525.0, - "99": 44279.0, - "100": 45708.0 + "1": 36436.0, + "2": 36501.0, + "3": 36802.0, + "4": 36766.0, + "5": 36386.0, + "6": 35733.0, + "7": 36883.0, + "8": 35982.0, + "9": 37134.0, + "10": 37619.0, + "11": 36856.0, + "12": 35393.0, + "13": 36987.0, + "14": 36916.0, + "15": 36126.0, + "16": 36007.0, + "17": 36166.0, + "18": 36642.0, + "19": 36691.0, + "20": 36267.0, + "21": 36325.0, + "22": 36849.0, + "23": 36559.0, + "24": 36807.0, + "25": 35927.0, + "26": 36377.0, + "27": 36395.0, + "28": 35441.0, + "29": 35838.0, + "30": 35285.0, + "31": 35827.0, + "32": 36688.0, + "33": 35140.0, + "34": 35614.0, + "35": 36283.0, + "36": 35123.0, + "37": 36078.0, + "38": 35203.0, + "39": 36835.0, + "40": 37651.0, + "41": 35925.0, + "42": 35887.0, + "43": 37261.0, + "44": 36196.0, + "45": 39547.0, + "46": 38230.0, + "47": 37972.0, + "48": 38405.0, + "49": 41846.0, + "50": 38991.0, + "51": 38781.0, + "52": 41557.0, + "53": 40236.0, + "54": 41428.0, + "55": 39279.0, + "56": 40896.0, + "57": 36831.0, + "58": 46014.0, + "59": 41870.0, + "60": 42002.0, + "61": 41281.0, + "62": 44376.0, + "63": 43888.0, + "64": 50350.0, + "65": 41693.0, + "66": 45434.0, + "67": 49882.0, + "68": 47180.0, + "69": 44903.0, + "70": 47813.0, + "71": 45797.0, + "72": 45119.0, + "73": 48038.0, + "74": 46007.0, + "75": 46998.0, + "76": 47058.0, + "77": 47604.0, + "78": 49536.0, + "79": 46549.0, + "80": 45509.0, + "81": 53807.0, + "82": 42629.0, + "83": 46031.0, + "84": 45537.0, + "85": 45231.0, + "86": 44892.0, + "87": 37088.0, + "88": 46123.0, + "89": 48687.0, + "90": 57077.0, + "91": 38387.0, + "92": 49670.0, + "93": 46990.0, + "94": 42760.0, + "95": 46020.0, + "96": 50224.0, + "97": 47737.0, + "98": 49761.0, + "99": 44729.0, + "100": 45627.0 } }, "mem-allocated-bytes": { @@ -218,105 +218,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254751744.0, - "2": 1254746624.0, - "3": 1254748160.0, - "4": 1254748160.0, + "1": 1254750720.0, + "2": 1254748160.0, + "3": 1254747136.0, + "4": 1254748672.0, "5": 1254748672.0, - "6": 1254751744.0, - "7": 1254750208.0, - "8": 1254747648.0, - "9": 1254748672.0, - "10": 1254747648.0, + "6": 1254752256.0, + "7": 1254750720.0, + "8": 1254747136.0, + "9": 1254747136.0, + "10": 1254746624.0, "11": 1254749184.0, - "12": 1254747136.0, - "13": 1254750720.0, + "12": 1254746624.0, + "13": 1254749696.0, "14": 1254748672.0, - "15": 1254748672.0, - "16": 1254746112.0, - "17": 1254747136.0, + "15": 1254747648.0, + "16": 1254747648.0, + "17": 1254748160.0, "18": 1254746624.0, - "19": 1254752256.0, + "19": 1254751744.0, "20": 1254746112.0, - "21": 1254748160.0, - "22": 1254749184.0, - "23": 1254749696.0, - "24": 1254747648.0, + "21": 1254747648.0, + "22": 1254748672.0, + "23": 1254749184.0, + "24": 1254747136.0, "25": 1254748160.0, - "26": 1254745600.0, + "26": 1254746624.0, "27": 1254749696.0, "28": 1254748160.0, - "29": 1254752768.0, - "30": 1254747648.0, - "31": 1254749184.0, - "32": 1254746112.0, + "29": 1254751744.0, + "30": 1254747136.0, + "31": 1254748160.0, + "32": 1254746624.0, "33": 1254748160.0, - "34": 1254747136.0, - "35": 1254746112.0, - "36": 1254748672.0, + "34": 1254745600.0, + "35": 1254747136.0, + "36": 1254748160.0, "37": 1254748160.0, - "38": 1254747648.0, - "39": 1254747136.0, - "40": 1254750720.0, - "41": 1254748160.0, - "42": 1254748672.0, - "43": 1254746624.0, - "44": 1254748160.0, - "45": 1254748160.0, - "46": 1254749184.0, - "47": 1254748672.0, - "48": 1254747648.0, - "49": 1254747648.0, - "50": 1254749696.0, - "51": 1254746624.0, - "52": 1254747648.0, - "53": 1254747136.0, + "38": 1254746624.0, + "39": 1254748160.0, + "40": 1254752256.0, + "41": 1254748672.0, + "42": 1254748160.0, + "43": 1254746112.0, + "44": 1254747648.0, + "45": 1254749696.0, + "46": 1254749696.0, + "47": 1254749696.0, + "48": 1254748160.0, + "49": 1254746624.0, + "50": 1254748672.0, + "51": 1254746112.0, + "52": 1254746624.0, + "53": 1254746624.0, "54": 1254747648.0, "55": 1254749184.0, - "56": 1254747136.0, + "56": 1254747648.0, "57": 1254749184.0, - "58": 1254749696.0, - "59": 1254747136.0, - "60": 1254745600.0, + "58": 1254749184.0, + "59": 1254746624.0, + "60": 1254745088.0, "61": 1254747136.0, - "62": 1254746624.0, - "63": 1254747136.0, + "62": 1254747136.0, + "63": 1254746112.0, "64": 1254748160.0, - "65": 1254747136.0, - "66": 1254750208.0, - "67": 1254745600.0, - "68": 1254749184.0, - "69": 1254746112.0, + "65": 1254747648.0, + "66": 1254749184.0, + "67": 1254746624.0, + "68": 1254750208.0, + "69": 1254745088.0, "70": 1254746112.0, - "71": 1254747136.0, + "71": 1254748160.0, "72": 1254750720.0, - "73": 1254746624.0, - "74": 1254746112.0, - "75": 1254749184.0, - "76": 1254748160.0, + "73": 1254747136.0, + "74": 1254745600.0, + "75": 1254748672.0, + "76": 1254747648.0, "77": 1254741504.0, - "78": 1254745600.0, - "79": 1254746112.0, + "78": 1254746112.0, + "79": 1254747648.0, "80": 1254745600.0, "81": 1254742528.0, - "82": 1254744064.0, - "83": 1254747136.0, - "84": 1254742016.0, - "85": 1254739968.0, - "86": 1254732288.0, - "87": 1254746112.0, - "88": 1254750208.0, - "89": 1254739456.0, + "82": 1254743040.0, + "83": 1254746624.0, + "84": 1254743040.0, + "85": 1254741504.0, + "86": 1254733312.0, + "87": 1254746624.0, + "88": 1254748160.0, + "89": 1254737920.0, "90": 1254739968.0, - "91": 1254746624.0, - "92": 1254736384.0, - "93": 1254741504.0, - "94": 1254736384.0, - "95": 1254732800.0, - "96": 1254748672.0, - "97": 1254734848.0, + "91": 1254747136.0, + "92": 1254735360.0, + "93": 1254742016.0, + "94": 1254737408.0, + "95": 1254733824.0, + "96": 1254747648.0, + "97": 1254734336.0, "98": 1254728192.0, - "99": 1254733312.0, + "99": 1254734848.0, "100": 1254732800.0 } }, @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2042493440.0, - "2": 2517739520.0, - "3": 2518202368.0, - "4": 2518202368.0, - "5": 2518202368.0, - "6": 2519861760.0, - "7": 2519861760.0, - "8": 2519861760.0, - "9": 2519861760.0, - "10": 2519861760.0, - "11": 2519861760.0, - "12": 2519861760.0, - "13": 2519861760.0, - "14": 2519861760.0, - "15": 2519861760.0, - "16": 2519861760.0, - "17": 2519861760.0, - "18": 2519861760.0, - "19": 2520205312.0, - "20": 2520205312.0, - "21": 2520205312.0, - "22": 2520205312.0, - "23": 2520205312.0, - "24": 2520205312.0, - "25": 2520205312.0, - "26": 2520205312.0, - "27": 2520205312.0, - "28": 2520205312.0, - "29": 2520205312.0, - "30": 2520205312.0, - "31": 2520205312.0, - "32": 2520205312.0, - "33": 2520205312.0, - "34": 2520205312.0, - "35": 2520205312.0, - "36": 2520205312.0, - "37": 2520205312.0, - "38": 2520205312.0, - "39": 2520205312.0, - "40": 2520205312.0, - "41": 2520205312.0, - "42": 2520205312.0, - "43": 2520205312.0, - "44": 2520205312.0, - "45": 2520205312.0, - "46": 2520205312.0, - "47": 2520205312.0, - "48": 2520205312.0, - "49": 2520205312.0, - "50": 2520205312.0, - "51": 2520205312.0, - "52": 2520205312.0, - "53": 2520205312.0, - "54": 2520205312.0, - "55": 2520205312.0, - "56": 2520205312.0, - "57": 2520205312.0, - "58": 2520205312.0, - "59": 2520205312.0, - "60": 2520205312.0, - "61": 2520205312.0, - "62": 2520205312.0, - "63": 2520205312.0, - "64": 2520205312.0, - "65": 2520205312.0, - "66": 2520205312.0, - "67": 2520205312.0, - "68": 2520205312.0, - "69": 2520205312.0, - "70": 2520205312.0, - "71": 2520205312.0, - "72": 2520205312.0, - "73": 2520205312.0, - "74": 2520205312.0, - "75": 2520205312.0, - "76": 2520205312.0, - "77": 2520205312.0, - "78": 2520205312.0, - "79": 2520205312.0, - "80": 2520205312.0, - "81": 2520205312.0, - "82": 2520205312.0, - "83": 2520205312.0, - "84": 2520205312.0, - "85": 2520205312.0, - "86": 2520205312.0, - "87": 2520205312.0, - "88": 2520205312.0, - "89": 2520205312.0, - "90": 2520205312.0, - "91": 2520205312.0, - "92": 2520205312.0, - "93": 2520205312.0, - "94": 2520205312.0, - "95": 2520205312.0, - "96": 2520205312.0, - "97": 2520205312.0, - "98": 2520205312.0, - "99": 2520205312.0, - "100": 2520205312.0 + "1": 2042772992.0, + "2": 2517462016.0, + "3": 2518086656.0, + "4": 2518086656.0, + "5": 2518086656.0, + "6": 2520246272.0, + "7": 2520246272.0, + "8": 2520246272.0, + "9": 2520246272.0, + "10": 2520246272.0, + "11": 2520246272.0, + "12": 2520246272.0, + "13": 2520246272.0, + "14": 2520246272.0, + "15": 2520246272.0, + "16": 2520246272.0, + "17": 2520246272.0, + "18": 2520246272.0, + "19": 2520246272.0, + "20": 2520246272.0, + "21": 2520246272.0, + "22": 2520246272.0, + "23": 2520246272.0, + "24": 2520246272.0, + "25": 2520246272.0, + "26": 2520246272.0, + "27": 2520246272.0, + "28": 2520246272.0, + "29": 2520246272.0, + "30": 2520246272.0, + "31": 2520246272.0, + "32": 2520246272.0, + "33": 2520246272.0, + "34": 2520246272.0, + "35": 2520246272.0, + "36": 2520246272.0, + "37": 2520246272.0, + "38": 2520246272.0, + "39": 2520246272.0, + "40": 2520246272.0, + "41": 2520246272.0, + "42": 2520246272.0, + "43": 2520246272.0, + "44": 2520246272.0, + "45": 2520246272.0, + "46": 2520246272.0, + "47": 2520246272.0, + "48": 2520246272.0, + "49": 2520246272.0, + "50": 2520246272.0, + "51": 2520246272.0, + "52": 2520246272.0, + "53": 2520246272.0, + "54": 2520246272.0, + "55": 2520246272.0, + "56": 2520246272.0, + "57": 2520246272.0, + "58": 2520246272.0, + "59": 2520246272.0, + "60": 2520246272.0, + "61": 2520246272.0, + "62": 2520246272.0, + "63": 2520246272.0, + "64": 2520246272.0, + "65": 2520246272.0, + "66": 2520246272.0, + "67": 2520246272.0, + "68": 2520246272.0, + "69": 2520246272.0, + "70": 2520246272.0, + "71": 2520246272.0, + "72": 2520246272.0, + "73": 2520246272.0, + "74": 2520246272.0, + "75": 2520246272.0, + "76": 2520246272.0, + "77": 2520246272.0, + "78": 2520246272.0, + "79": 2520246272.0, + "80": 2520246272.0, + "81": 2520246272.0, + "82": 2520246272.0, + "83": 2520246272.0, + "84": 2520246272.0, + "85": 2520246272.0, + "86": 2520246272.0, + "87": 2520246272.0, + "88": 2520246272.0, + "89": 2520246272.0, + "90": 2520246272.0, + "91": 2520246272.0, + "92": 2520246272.0, + "93": 2520246272.0, + "94": 2520246272.0, + "95": 2520246272.0, + "96": 2520246272.0, + "97": 2520246272.0, + "98": 2520246272.0, + "99": 2520246272.0, + "100": 2520246272.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.6311, - "3": 0.34592, - "4": 0.34521, - "5": 0.32602, - "6": 0.32592, - "7": 0.33225, - "8": 0.32209, - "9": 0.31799, - "10": 0.32175, - "11": 0.31963, - "12": 0.31948, - "13": 0.32628, - "14": 0.32525, - "15": 0.32035, - "16": 0.31631, - "17": 0.3172, - "18": 0.31752, - "19": 0.31362, - "20": 0.3134, - "21": 0.3139, - "22": 0.31635, - "23": 0.31613, - "24": 0.31472, - "25": 0.31973, - "26": 0.31472, - "27": 0.31887, - "28": 0.31268, - "29": 0.31511, - "30": 0.31604, - "31": 0.31415, - "32": 0.31381, - "33": 0.31328, - "34": 0.3143, - "35": 0.3122, - "36": 0.30943, - "37": 0.31565, - "38": 0.31793, - "39": 0.32043, - "40": 0.32101, - "41": 0.31783, - "42": 0.31527, - "43": 0.31523, - "44": 0.31606, - "45": 0.31171, - "46": 0.31406, - "47": 0.31657, - "48": 0.32111, - "49": 0.31335, - "50": 0.32346, - "51": 0.37683, - "52": 0.38404, - "53": 0.32399, - "54": 0.32106, - "55": 0.32305, - "56": 0.31666, - "57": 0.31622, - "58": 0.32015, - "59": 0.31487, - "60": 0.32232, - "61": 0.3198, - "62": 0.31879, - "63": 0.31919, - "64": 0.86198, - "65": 0.32672, - "66": 0.31588, - "67": 0.31858, - "68": 0.32169, - "69": 0.31983, - "70": 0.31958, - "71": 0.32528, - "72": 0.3201, - "73": 0.32347, - "74": 0.33992, - "75": 0.33753, - "76": 0.32343, - "77": 0.34573, - "78": 0.33165, - "79": 0.32229, - "80": 0.33411, - "81": 0.32778, - "82": 0.33225, - "83": 0.36857, - "84": 0.34612, - "85": 0.34462, - "86": 0.34803, - "87": 0.32667, - "88": 0.32339, - "89": 0.35636, - "90": 0.33681, - "91": 0.3477, - "92": 0.33318, - "93": 0.35303, - "94": 0.33954, - "95": 0.3624, - "96": 0.38791, - "97": 0.36171, - "98": 0.35126, - "99": 0.33264, - "100": 0.34722 + "2": 6.43348, + "3": 0.34868, + "4": 0.35485, + "5": 0.34067, + "6": 0.33734, + "7": 0.3405, + "8": 0.33604, + "9": 0.33279, + "10": 0.33823, + "11": 0.33044, + "12": 0.33064, + "13": 0.32902, + "14": 0.33854, + "15": 0.33408, + "16": 0.33249, + "17": 0.32644, + "18": 0.32811, + "19": 0.32913, + "20": 0.32285, + "21": 0.3224, + "22": 0.32781, + "23": 0.33028, + "24": 0.32228, + "25": 0.32687, + "26": 0.33064, + "27": 0.33312, + "28": 0.32337, + "29": 0.32345, + "30": 0.32953, + "31": 0.32669, + "32": 0.32682, + "33": 0.32322, + "34": 0.32625, + "35": 0.32534, + "36": 0.32236, + "37": 0.32355, + "38": 0.33148, + "39": 0.32994, + "40": 0.3237, + "41": 0.33222, + "42": 0.32815, + "43": 0.32785, + "44": 0.32579, + "45": 0.31998, + "46": 0.32769, + "47": 0.32148, + "48": 0.32802, + "49": 0.3203, + "50": 0.3278, + "51": 0.36598, + "52": 0.38189, + "53": 0.33445, + "54": 0.3391, + "55": 0.32893, + "56": 0.32847, + "57": 0.32895, + "58": 0.33097, + "59": 0.32595, + "60": 0.32463, + "61": 0.33107, + "62": 0.34742, + "63": 0.32734, + "64": 0.32489, + "65": 0.32857, + "66": 0.32706, + "67": 0.32589, + "68": 0.32685, + "69": 0.33064, + "70": 0.32853, + "71": 0.33369, + "72": 0.32627, + "73": 0.33692, + "74": 0.97285, + "75": 0.34219, + "76": 0.33076, + "77": 0.34507, + "78": 0.34, + "79": 0.36121, + "80": 0.3495, + "81": 0.33341, + "82": 0.33575, + "83": 0.3854, + "84": 0.35574, + "85": 0.35951, + "86": 0.35992, + "87": 0.32721, + "88": 0.35021, + "89": 0.3738, + "90": 0.35188, + "91": 0.34121, + "92": 0.35643, + "93": 0.36296, + "94": 0.36373, + "95": 0.37232, + "96": 0.38213, + "97": 0.37872, + "98": 0.36666, + "99": 0.35693, + "100": 0.37117 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json index fe59d4e351b..cfbdd7d4495 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92456, - "2": 10.91122, - "3": 10.92478, - "4": 10.91392, - "5": 10.91847, - "6": 10.90864, - "7": 10.90181, - "8": 10.90918, - "9": 10.92285, - "10": 10.91171, - "11": 10.90188, - "12": 10.91191, - "13": 10.88926, - "14": 10.8908, - "15": 10.88438, - "16": 10.87546, - "17": 10.87471, - "18": 10.8628, - "19": 10.8724, - "20": 10.8201, - "21": 10.8072, - "22": 10.79503, - "23": 10.79079, - "24": 10.75741, - "25": 10.76214, - "26": 10.75151, - "27": 10.73656, - "28": 10.67545, - "29": 10.63996, - "30": 10.62263, - "31": 10.61218, - "32": 10.59655, - "33": 10.5765, - "34": 10.53881, - "35": 10.53788, - "36": 10.5216, - "37": 10.48667, - "38": 10.49899, - "39": 10.45883, - "40": 10.44007, - "41": 10.42695, - "42": 10.41344, - "43": 10.38989, - "44": 10.35254, - "45": 10.36981, - "46": 10.33007, - "47": 10.31289, - "48": 10.27798, - "49": 10.27103, - "50": 10.27187 + "1": 10.92337, + "2": 10.91184, + "3": 10.9245, + "4": 10.91413, + "5": 10.91791, + "6": 10.90788, + "7": 10.90285, + "8": 10.9094, + "9": 10.92264, + "10": 10.91325, + "11": 10.90217, + "12": 10.91139, + "13": 10.88971, + "14": 10.88954, + "15": 10.88501, + "16": 10.87598, + "17": 10.87426, + "18": 10.8634, + "19": 10.87212, + "20": 10.82076, + "21": 10.80748, + "22": 10.79544, + "23": 10.79148, + "24": 10.75643, + "25": 10.76142, + "26": 10.75154, + "27": 10.73598, + "28": 10.67469, + "29": 10.64016, + "30": 10.62245, + "31": 10.61276, + "32": 10.59683, + "33": 10.57641, + "34": 10.53937, + "35": 10.53846, + "36": 10.52255, + "37": 10.48675, + "38": 10.49927, + "39": 10.45887, + "40": 10.43983, + "41": 10.42774, + "42": 10.41415, + "43": 10.3903, + "44": 10.35238, + "45": 10.36987, + "46": 10.33008, + "47": 10.31244, + "48": 10.27782, + "49": 10.27035, + "50": 10.27145 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18865.0, - "2": 18932.0, - "3": 18832.0, - "4": 18774.0, - "5": 18548.0, - "6": 18607.0, - "7": 18950.0, - "8": 18828.0, - "9": 19101.0, - "10": 19425.0, - "11": 19225.0, - "12": 18243.0, - "13": 19160.0, - "14": 19145.0, - "15": 18772.0, - "16": 18768.0, - "17": 18807.0, - "18": 19111.0, - "19": 18890.0, - "20": 19015.0, - "21": 18479.0, - "22": 18922.0, - "23": 18880.0, - "24": 18812.0, - "25": 18185.0, - "26": 18922.0, - "27": 18466.0, - "28": 18402.0, - "29": 18510.0, - "30": 18253.0, - "31": 18896.0, - "32": 18852.0, - "33": 18338.0, - "34": 18553.0, - "35": 18775.0, - "36": 18344.0, - "37": 19084.0, - "38": 18477.0, - "39": 18845.0, - "40": 19477.0, - "41": 18606.0, - "42": 18524.0, - "43": 19412.0, - "44": 18862.0, - "45": 20342.0, - "46": 19670.0, - "47": 19622.0, - "48": 19999.0, - "49": 21560.0, - "50": 20111.0 + "1": 18811.0, + "2": 19165.0, + "3": 19169.0, + "4": 18794.0, + "5": 18806.0, + "6": 18547.0, + "7": 18947.0, + "8": 18963.0, + "9": 18914.0, + "10": 19344.0, + "11": 18881.0, + "12": 18329.0, + "13": 18937.0, + "14": 18987.0, + "15": 18708.0, + "16": 18957.0, + "17": 19271.0, + "18": 18855.0, + "19": 18843.0, + "20": 19091.0, + "21": 18628.0, + "22": 18905.0, + "23": 18905.0, + "24": 19000.0, + "25": 18306.0, + "26": 18509.0, + "27": 18924.0, + "28": 18259.0, + "29": 18452.0, + "30": 18270.0, + "31": 18685.0, + "32": 18898.0, + "33": 18250.0, + "34": 18394.0, + "35": 18736.0, + "36": 18582.0, + "37": 18806.0, + "38": 18396.0, + "39": 19248.0, + "40": 19332.0, + "41": 18568.0, + "42": 18144.0, + "43": 19243.0, + "44": 18642.0, + "45": 20136.0, + "46": 19403.0, + "47": 19891.0, + "48": 20076.0, + "49": 21584.0, + "50": 20152.0 } }, "mem-allocated-bytes": { @@ -118,55 +118,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388730880.0, - "2": 1388731392.0, + "1": 1388729856.0, + "2": 1388730368.0, "3": 1388729856.0, - "4": 1388727296.0, - "5": 1388729856.0, + "4": 1388727808.0, + "5": 1388728832.0, "6": 1388730368.0, - "7": 1388729856.0, + "7": 1388731904.0, "8": 1388729856.0, - "9": 1388731392.0, + "9": 1388731904.0, "10": 1388731392.0, - "11": 1388729344.0, + "11": 1388728832.0, "12": 1388728832.0, "13": 1388730880.0, "14": 1388728320.0, - "15": 1388730880.0, - "16": 1388729344.0, - "17": 1388731904.0, - "18": 1388730880.0, - "19": 1388730368.0, + "15": 1388731904.0, + "16": 1388728320.0, + "17": 1388730880.0, + "18": 1388730368.0, + "19": 1388729856.0, "20": 1388730368.0, - "21": 1388731392.0, - "22": 1388730880.0, + "21": 1388732928.0, + "22": 1388729344.0, "23": 1388731392.0, - "24": 1388730880.0, - "25": 1388731904.0, + "24": 1388729344.0, + "25": 1388732928.0, "26": 1388732928.0, - "27": 1388731392.0, - "28": 1388729856.0, + "27": 1388729344.0, + "28": 1388730368.0, "29": 1388731392.0, "30": 1388733440.0, "31": 1388732416.0, - "32": 1388730368.0, + "32": 1388729344.0, "33": 1388732416.0, "34": 1388734464.0, - "35": 1388731904.0, + "35": 1388731392.0, "36": 1388730368.0, "37": 1388731904.0, - "38": 1388729856.0, - "39": 1388732928.0, - "40": 1388733952.0, - "41": 1388729856.0, + "38": 1388730368.0, + "39": 1388733440.0, + "40": 1388734464.0, + "41": 1388730368.0, "42": 1388732928.0, "43": 1388731392.0, "44": 1388729856.0, - "45": 1388733952.0, - "46": 1388731392.0, - "47": 1388732928.0, + "45": 1388732928.0, + "46": 1388731904.0, + "47": 1388733952.0, "48": 1388731904.0, - "49": 1388736000.0, + "49": 1388736512.0, "50": 1388731904.0 } }, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179614208.0, - "2": 3661343744.0, - "3": 3663287296.0, - "4": 3663287296.0, - "5": 3663287296.0, - "6": 3663287296.0, - "7": 3663287296.0, - "8": 3663287296.0, - "9": 3663287296.0, - "10": 3663287296.0, - "11": 3663287296.0, - "12": 3663287296.0, - "13": 3663287296.0, - "14": 3663287296.0, - "15": 3663287296.0, - "16": 3663287296.0, - "17": 3663287296.0, - "18": 3663287296.0, - "19": 3663287296.0, - "20": 3663287296.0, - "21": 3663287296.0, - "22": 3663287296.0, - "23": 3663287296.0, - "24": 3663287296.0, - "25": 3663287296.0, - "26": 3663287296.0, - "27": 3663287296.0, - "28": 3663287296.0, - "29": 3663287296.0, - "30": 3663287296.0, - "31": 3663287296.0, - "32": 3663287296.0, - "33": 3663287296.0, - "34": 3664704512.0, - "35": 3664704512.0, - "36": 3664704512.0, - "37": 3664704512.0, - "38": 3664704512.0, - "39": 3664704512.0, - "40": 3664704512.0, - "41": 3664704512.0, - "42": 3664704512.0, - "43": 3664704512.0, - "44": 3664704512.0, - "45": 3664704512.0, - "46": 3665018368.0, - "47": 3665018368.0, - "48": 3665595904.0, - "49": 3665595904.0, - "50": 3665595904.0 + "1": 3179474432.0, + "2": 3662153216.0, + "3": 3662153216.0, + "4": 3662153216.0, + "5": 3662153216.0, + "6": 3662153216.0, + "7": 3662153216.0, + "8": 3662153216.0, + "9": 3662153216.0, + "10": 3662261248.0, + "11": 3662261248.0, + "12": 3662261248.0, + "13": 3662261248.0, + "14": 3662261248.0, + "15": 3662261248.0, + "16": 3662261248.0, + "17": 3662261248.0, + "18": 3662261248.0, + "19": 3662745600.0, + "20": 3662745600.0, + "21": 3662745600.0, + "22": 3662745600.0, + "23": 3662745600.0, + "24": 3662745600.0, + "25": 3662901760.0, + "26": 3663353344.0, + "27": 3663353344.0, + "28": 3663353344.0, + "29": 3663353344.0, + "30": 3663353344.0, + "31": 3663353344.0, + "32": 3663353344.0, + "33": 3663353344.0, + "34": 3663353344.0, + "35": 3663353344.0, + "36": 3663353344.0, + "37": 3663353344.0, + "38": 3663353344.0, + "39": 3664240640.0, + "40": 3664613376.0, + "41": 3664613376.0, + "42": 3664613376.0, + "43": 3664613376.0, + "44": 3664749056.0, + "45": 3664749056.0, + "46": 3664804352.0, + "47": 3664804352.0, + "48": 3664804352.0, + "49": 3664804352.0, + "50": 3664804352.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.32018, - "3": 1.74495, - "4": 1.64524, - "5": 0.85018, - "6": 1.50819, - "7": 1.44545, - "8": 1.66946, - "9": 1.72718, - "10": 1.18171, - "11": 1.35307, - "12": 1.46593, - "13": 1.24052, - "14": 1.87146, - "15": 0.99226, - "16": 1.29853, - "17": 1.35074, - "18": 1.36129, - "19": 1.39156, - "20": 1.72658, - "21": 1.32313, - "22": 1.72423, - "23": 1.72372, - "24": 2.64381, - "25": 1.46675, - "26": 2.26793, - "27": 1.80467, - "28": 3.05207, - "29": 2.34622, - "30": 1.46261, - "31": 2.24908, - "32": 2.42053, - "33": 1.30876, - "34": 1.44075, - "35": 2.05636, - "36": 1.56688, - "37": 1.6821, - "38": 1.48888, - "39": 1.57894, - "40": 1.92345, - "41": 1.46836, - "42": 2.77498, - "43": 1.72973, - "44": 2.152, - "45": 0.78685, - "46": 1.53871, - "47": 2.04434, - "48": 1.7392, - "49": 1.41271, - "50": 2.00804 + "2": 5.09198, + "3": 1.27328, + "4": 1.08714, + "5": 0.74338, + "6": 1.43552, + "7": 1.39107, + "8": 1.51041, + "9": 1.60025, + "10": 1.15342, + "11": 1.22596, + "12": 1.25577, + "13": 1.10285, + "14": 1.56763, + "15": 0.95369, + "16": 1.1225, + "17": 1.53102, + "18": 1.39185, + "19": 1.3467, + "20": 1.13977, + "21": 1.11561, + "22": 1.04201, + "23": 1.12439, + "24": 1.63221, + "25": 0.65992, + "26": 1.04825, + "27": 1.12471, + "28": 1.02379, + "29": 1.07212, + "30": 0.92595, + "31": 0.82823, + "32": 1.98295, + "33": 1.08839, + "34": 0.57042, + "35": 1.34861, + "36": 1.2162, + "37": 0.98446, + "38": 0.58031, + "39": 1.19166, + "40": 1.37951, + "41": 1.03137, + "42": 1.14312, + "43": 1.02217, + "44": 1.3242, + "45": 0.64921, + "46": 1.02878, + "47": 1.06606, + "48": 1.0625, + "49": 0.97755, + "50": 1.14391 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json index fbaf488494a..a9e21e02df3 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.92456, - "2": 10.91122, - "3": 10.92478, - "4": 10.91392, - "5": 10.91847, - "6": 10.90864, - "7": 10.90181, - "8": 10.90918, - "9": 10.92285, - "10": 10.91171, - "11": 10.90188, - "12": 10.91191, - "13": 10.88926, - "14": 10.8908, - "15": 10.88438, - "16": 10.87546, - "17": 10.87471, - "18": 10.8628, - "19": 10.8724, - "20": 10.8201, - "21": 10.8072, - "22": 10.79503, - "23": 10.79079, - "24": 10.75741, - "25": 10.76214, - "26": 10.75151, - "27": 10.73656, - "28": 10.67545, - "29": 10.63996, - "30": 10.62263, - "31": 10.61218, - "32": 10.59655, - "33": 10.5765, - "34": 10.53881, - "35": 10.53788, - "36": 10.5216, - "37": 10.48667, - "38": 10.49899, - "39": 10.45883, - "40": 10.44007, - "41": 10.42695, - "42": 10.41344, - "43": 10.38989, - "44": 10.35254, - "45": 10.36981, - "46": 10.33007, - "47": 10.31289, - "48": 10.27798, - "49": 10.27103, - "50": 10.27187 + "1": 10.92337, + "2": 10.91184, + "3": 10.9245, + "4": 10.91413, + "5": 10.91791, + "6": 10.90788, + "7": 10.90285, + "8": 10.9094, + "9": 10.92264, + "10": 10.91325, + "11": 10.90217, + "12": 10.91139, + "13": 10.88971, + "14": 10.88954, + "15": 10.88501, + "16": 10.87598, + "17": 10.87426, + "18": 10.8634, + "19": 10.87212, + "20": 10.82076, + "21": 10.80748, + "22": 10.79544, + "23": 10.79148, + "24": 10.75643, + "25": 10.76142, + "26": 10.75154, + "27": 10.73598, + "28": 10.67469, + "29": 10.64016, + "30": 10.62245, + "31": 10.61276, + "32": 10.59683, + "33": 10.57641, + "34": 10.53937, + "35": 10.53846, + "36": 10.52255, + "37": 10.48675, + "38": 10.49927, + "39": 10.45887, + "40": 10.43983, + "41": 10.42774, + "42": 10.41415, + "43": 10.3903, + "44": 10.35238, + "45": 10.36987, + "46": 10.33008, + "47": 10.31244, + "48": 10.27782, + "49": 10.27035, + "50": 10.27145 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18865.0, - "2": 18932.0, - "3": 18832.0, - "4": 18774.0, - "5": 18548.0, - "6": 18607.0, - "7": 18950.0, - "8": 18828.0, - "9": 19101.0, - "10": 19425.0, - "11": 19225.0, - "12": 18243.0, - "13": 19160.0, - "14": 19145.0, - "15": 18772.0, - "16": 18768.0, - "17": 18807.0, - "18": 19111.0, - "19": 18890.0, - "20": 19015.0, - "21": 18479.0, - "22": 18922.0, - "23": 18880.0, - "24": 18812.0, - "25": 18185.0, - "26": 18922.0, - "27": 18466.0, - "28": 18402.0, - "29": 18510.0, - "30": 18253.0, - "31": 18896.0, - "32": 18852.0, - "33": 18338.0, - "34": 18553.0, - "35": 18775.0, - "36": 18344.0, - "37": 19084.0, - "38": 18477.0, - "39": 18845.0, - "40": 19477.0, - "41": 18606.0, - "42": 18524.0, - "43": 19412.0, - "44": 18862.0, - "45": 20342.0, - "46": 19670.0, - "47": 19622.0, - "48": 19999.0, - "49": 21560.0, - "50": 20111.0 + "1": 18811.0, + "2": 19165.0, + "3": 19169.0, + "4": 18794.0, + "5": 18806.0, + "6": 18547.0, + "7": 18947.0, + "8": 18963.0, + "9": 18914.0, + "10": 19344.0, + "11": 18881.0, + "12": 18329.0, + "13": 18937.0, + "14": 18987.0, + "15": 18708.0, + "16": 18957.0, + "17": 19271.0, + "18": 18855.0, + "19": 18843.0, + "20": 19091.0, + "21": 18628.0, + "22": 18905.0, + "23": 18905.0, + "24": 19000.0, + "25": 18306.0, + "26": 18509.0, + "27": 18924.0, + "28": 18259.0, + "29": 18452.0, + "30": 18270.0, + "31": 18685.0, + "32": 18898.0, + "33": 18250.0, + "34": 18394.0, + "35": 18736.0, + "36": 18582.0, + "37": 18806.0, + "38": 18396.0, + "39": 19248.0, + "40": 19332.0, + "41": 18568.0, + "42": 18144.0, + "43": 19243.0, + "44": 18642.0, + "45": 20136.0, + "46": 19403.0, + "47": 19891.0, + "48": 20076.0, + "49": 21584.0, + "50": 20152.0 } }, "mem-allocated-bytes": { @@ -118,55 +118,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388730880.0, - "2": 1388731392.0, + "1": 1388729856.0, + "2": 1388730368.0, "3": 1388729856.0, - "4": 1388727296.0, - "5": 1388729856.0, + "4": 1388727808.0, + "5": 1388728832.0, "6": 1388730368.0, - "7": 1388729856.0, + "7": 1388731904.0, "8": 1388729856.0, - "9": 1388731392.0, + "9": 1388731904.0, "10": 1388731392.0, - "11": 1388729344.0, + "11": 1388728832.0, "12": 1388728832.0, "13": 1388730880.0, "14": 1388728320.0, - "15": 1388730880.0, - "16": 1388729344.0, - "17": 1388731904.0, - "18": 1388730880.0, - "19": 1388730368.0, + "15": 1388731904.0, + "16": 1388728320.0, + "17": 1388730880.0, + "18": 1388730368.0, + "19": 1388729856.0, "20": 1388730368.0, - "21": 1388731392.0, - "22": 1388730880.0, + "21": 1388732928.0, + "22": 1388729344.0, "23": 1388731392.0, - "24": 1388730880.0, - "25": 1388731904.0, + "24": 1388729344.0, + "25": 1388732928.0, "26": 1388732928.0, - "27": 1388731392.0, - "28": 1388729856.0, + "27": 1388729344.0, + "28": 1388730368.0, "29": 1388731392.0, "30": 1388733440.0, "31": 1388732416.0, - "32": 1388730368.0, + "32": 1388729344.0, "33": 1388732416.0, "34": 1388734464.0, - "35": 1388731904.0, + "35": 1388731392.0, "36": 1388730368.0, "37": 1388731904.0, - "38": 1388729856.0, - "39": 1388732928.0, - "40": 1388733952.0, - "41": 1388729856.0, + "38": 1388730368.0, + "39": 1388733440.0, + "40": 1388734464.0, + "41": 1388730368.0, "42": 1388732928.0, "43": 1388731392.0, "44": 1388729856.0, - "45": 1388733952.0, - "46": 1388731392.0, - "47": 1388732928.0, + "45": 1388732928.0, + "46": 1388731904.0, + "47": 1388733952.0, "48": 1388731904.0, - "49": 1388736000.0, + "49": 1388736512.0, "50": 1388731904.0 } }, @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179614208.0, - "2": 3661343744.0, - "3": 3663287296.0, - "4": 3663287296.0, - "5": 3663287296.0, - "6": 3663287296.0, - "7": 3663287296.0, - "8": 3663287296.0, - "9": 3663287296.0, - "10": 3663287296.0, - "11": 3663287296.0, - "12": 3663287296.0, - "13": 3663287296.0, - "14": 3663287296.0, - "15": 3663287296.0, - "16": 3663287296.0, - "17": 3663287296.0, - "18": 3663287296.0, - "19": 3663287296.0, - "20": 3663287296.0, - "21": 3663287296.0, - "22": 3663287296.0, - "23": 3663287296.0, - "24": 3663287296.0, - "25": 3663287296.0, - "26": 3663287296.0, - "27": 3663287296.0, - "28": 3663287296.0, - "29": 3663287296.0, - "30": 3663287296.0, - "31": 3663287296.0, - "32": 3663287296.0, - "33": 3663287296.0, - "34": 3664704512.0, - "35": 3664704512.0, - "36": 3664704512.0, - "37": 3664704512.0, - "38": 3664704512.0, - "39": 3664704512.0, - "40": 3664704512.0, - "41": 3664704512.0, - "42": 3664704512.0, - "43": 3664704512.0, - "44": 3664704512.0, - "45": 3664704512.0, - "46": 3665018368.0, - "47": 3665018368.0, - "48": 3665595904.0, - "49": 3665595904.0, - "50": 3665595904.0 + "1": 3179474432.0, + "2": 3662153216.0, + "3": 3662153216.0, + "4": 3662153216.0, + "5": 3662153216.0, + "6": 3662153216.0, + "7": 3662153216.0, + "8": 3662153216.0, + "9": 3662153216.0, + "10": 3662261248.0, + "11": 3662261248.0, + "12": 3662261248.0, + "13": 3662261248.0, + "14": 3662261248.0, + "15": 3662261248.0, + "16": 3662261248.0, + "17": 3662261248.0, + "18": 3662261248.0, + "19": 3662745600.0, + "20": 3662745600.0, + "21": 3662745600.0, + "22": 3662745600.0, + "23": 3662745600.0, + "24": 3662745600.0, + "25": 3662901760.0, + "26": 3663353344.0, + "27": 3663353344.0, + "28": 3663353344.0, + "29": 3663353344.0, + "30": 3663353344.0, + "31": 3663353344.0, + "32": 3663353344.0, + "33": 3663353344.0, + "34": 3663353344.0, + "35": 3663353344.0, + "36": 3663353344.0, + "37": 3663353344.0, + "38": 3663353344.0, + "39": 3664240640.0, + "40": 3664613376.0, + "41": 3664613376.0, + "42": 3664613376.0, + "43": 3664613376.0, + "44": 3664749056.0, + "45": 3664749056.0, + "46": 3664804352.0, + "47": 3664804352.0, + "48": 3664804352.0, + "49": 3664804352.0, + "50": 3664804352.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.34092, - "3": 1.58696, - "4": 1.54545, - "5": 0.82593, - "6": 1.57399, - "7": 1.36079, - "8": 1.55567, - "9": 1.67303, - "10": 1.17161, - "11": 1.37021, - "12": 1.29365, - "13": 1.13607, - "14": 1.7201, - "15": 0.98492, - "16": 1.29971, - "17": 1.38451, - "18": 1.29949, - "19": 1.23528, - "20": 1.32649, - "21": 1.12039, - "22": 1.20629, - "23": 1.29693, - "24": 2.1645, - "25": 0.71347, - "26": 1.44727, - "27": 1.23095, - "28": 1.01048, - "29": 1.37169, - "30": 1.09444, - "31": 0.92601, - "32": 1.99775, - "33": 1.07593, - "34": 0.64632, - "35": 1.48535, - "36": 1.51704, - "37": 1.27474, - "38": 0.76789, - "39": 1.54033, - "40": 1.66902, - "41": 1.19572, - "42": 1.46501, - "43": 1.15658, - "44": 1.90014, - "45": 0.77048, - "46": 1.07968, - "47": 1.28398, - "48": 1.29999, - "49": 1.10163, - "50": 1.3568 + "2": 5.3448, + "3": 1.72915, + "4": 1.56321, + "5": 0.96252, + "6": 1.48635, + "7": 1.44339, + "8": 1.74183, + "9": 2.09358, + "10": 0.88137, + "11": 1.47152, + "12": 1.31738, + "13": 1.15672, + "14": 1.69174, + "15": 1.15258, + "16": 1.16002, + "17": 1.48684, + "18": 1.4527, + "19": 1.36347, + "20": 1.21791, + "21": 1.21718, + "22": 1.25722, + "23": 1.33315, + "24": 1.81753, + "25": 0.80591, + "26": 1.15336, + "27": 1.28463, + "28": 1.14105, + "29": 1.23623, + "30": 1.01784, + "31": 0.86463, + "32": 1.91821, + "33": 1.16394, + "34": 0.58055, + "35": 1.41154, + "36": 1.40364, + "37": 1.13305, + "38": 0.68315, + "39": 1.48977, + "40": 1.60313, + "41": 1.12241, + "42": 1.12332, + "43": 0.87168, + "44": 1.3132, + "45": 1.03733, + "46": 0.73916, + "47": 1.17411, + "48": 1.0725, + "49": 1.06652, + "50": 1.19903 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json index e5622d351fb..a9c20e6956c 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_1node/golden_values_dev_dgx_gb200.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.91962, - "2": 10.9168, - "3": 10.9211, - "4": 10.9202, - "5": 10.92774, - "6": 10.91636, - "7": 10.91825, - "8": 10.91825, - "9": 10.91448, - "10": 10.91644, - "11": 10.90375, - "12": 10.91532, - "13": 10.89542, - "14": 10.89837, - "15": 10.88246, - "16": 10.86562, - "17": 10.86327, - "18": 10.87059, - "19": 10.86132, - "20": 10.79564, - "21": 10.78115, - "22": 10.76223, - "23": 10.75418, - "24": 10.71745, - "25": 10.71507, - "26": 10.70251, - "27": 10.68134, - "28": 10.61348, - "29": 10.5756, - "30": 10.55196, - "31": 10.54825, - "32": 10.51901, - "33": 10.48575, - "34": 10.45833, - "35": 10.46344, - "36": 10.44142, - "37": 10.39933, - "38": 10.40604, - "39": 10.36502, - "40": 10.35425, - "41": 10.33201, - "42": 10.32538, - "43": 10.30184, - "44": 10.26353, - "45": 10.28155, - "46": 10.24406, - "47": 10.23118, - "48": 10.19372, - "49": 10.18577, - "50": 10.1917 + "1": 10.91966, + "2": 10.91726, + "3": 10.92132, + "4": 10.92035, + "5": 10.92817, + "6": 10.91641, + "7": 10.91804, + "8": 10.91811, + "9": 10.91453, + "10": 10.91661, + "11": 10.90391, + "12": 10.91493, + "13": 10.89584, + "14": 10.89893, + "15": 10.88207, + "16": 10.86474, + "17": 10.86288, + "18": 10.87074, + "19": 10.8622, + "20": 10.79575, + "21": 10.78195, + "22": 10.76221, + "23": 10.75467, + "24": 10.7176, + "25": 10.71567, + "26": 10.70228, + "27": 10.68101, + "28": 10.61305, + "29": 10.5759, + "30": 10.55078, + "31": 10.54866, + "32": 10.51944, + "33": 10.4861, + "34": 10.45799, + "35": 10.46346, + "36": 10.44157, + "37": 10.39981, + "38": 10.40598, + "39": 10.36498, + "40": 10.35539, + "41": 10.332, + "42": 10.32524, + "43": 10.30185, + "44": 10.26362, + "45": 10.28123, + "46": 10.24404, + "47": 10.23119, + "48": 10.19333, + "49": 10.18598, + "50": 10.19193 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 16504.0, - "2": 16572.0, - "3": 16848.0, - "4": 16537.0, - "5": 16489.0, - "6": 16124.0, - "7": 16810.0, - "8": 16150.0, - "9": 16548.0, - "10": 17006.0, - "11": 16733.0, - "12": 16149.0, - "13": 16785.0, - "14": 16595.0, - "15": 15935.0, - "16": 16182.0, - "17": 16600.0, - "18": 16851.0, - "19": 16582.0, - "20": 16488.0, - "21": 16536.0, - "22": 16707.0, - "23": 16806.0, - "24": 17135.0, - "25": 16344.0, - "26": 16797.0, - "27": 16755.0, - "28": 16541.0, - "29": 16348.0, - "30": 16485.0, - "31": 17033.0, - "32": 17299.0, - "33": 17333.0, - "34": 17176.0, - "35": 17567.0, - "36": 17050.0, - "37": 17719.0, - "38": 17231.0, - "39": 18134.0, - "40": 18748.0, - "41": 17574.0, - "42": 17132.0, - "43": 18791.0, - "44": 18164.0, - "45": 20261.0, - "46": 19601.0, - "47": 19299.0, - "48": 19756.0, - "49": 22082.0, - "50": 20481.0 + "1": 16524.0, + "2": 16508.0, + "3": 16785.0, + "4": 16926.0, + "5": 16462.0, + "6": 16102.0, + "7": 16748.0, + "8": 16199.0, + "9": 16613.0, + "10": 16943.0, + "11": 16425.0, + "12": 16180.0, + "13": 16657.0, + "14": 16782.0, + "15": 16148.0, + "16": 16106.0, + "17": 16492.0, + "18": 16533.0, + "19": 16530.0, + "20": 16763.0, + "21": 16445.0, + "22": 16773.0, + "23": 16630.0, + "24": 16701.0, + "25": 16064.0, + "26": 17085.0, + "27": 16732.0, + "28": 16456.0, + "29": 16464.0, + "30": 16595.0, + "31": 16897.0, + "32": 17338.0, + "33": 17253.0, + "34": 16885.0, + "35": 17615.0, + "36": 17281.0, + "37": 17687.0, + "38": 17043.0, + "39": 18394.0, + "40": 18808.0, + "41": 17518.0, + "42": 17212.0, + "43": 19063.0, + "44": 17953.0, + "45": 20054.0, + "46": 19544.0, + "47": 19211.0, + "48": 19825.0, + "49": 22059.0, + "50": 20151.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1561137664.0, - "2": 1561081856.0, - "3": 1561233920.0, - "4": 1561094144.0, - "5": 1561165312.0, - "6": 1561183232.0, - "7": 1562110464.0, - "8": 1562260480.0, - "9": 1561183232.0, - "10": 1561138176.0, - "11": 1561670144.0, - "12": 1561515520.0, - "13": 1561385984.0, - "14": 1561284608.0, - "15": 1561436672.0, - "16": 1561284608.0, - "17": 1562002944.0, - "18": 1561442304.0, - "19": 1561391616.0, - "20": 1561487360.0, - "21": 1561539584.0, - "22": 1561284608.0, - "23": 1561492992.0, - "24": 1561492992.0, - "25": 1561715712.0, - "26": 1561487360.0, - "27": 1562130944.0, - "28": 1561621504.0, - "29": 1562286592.0, - "30": 1562626048.0, - "31": 1562769920.0, - "32": 1561718784.0, - "33": 1561633792.0, - "34": 1561487360.0, - "35": 1561639424.0, - "36": 1561639424.0, - "37": 1561791488.0, - "38": 1561690112.0, - "39": 1561842176.0, - "40": 1561832448.0, - "41": 1563395584.0, - "42": 1561847808.0, - "43": 1563095552.0, - "44": 1562469376.0, - "45": 1561847808.0, - "46": 1561931776.0, - "47": 1561898496.0, - "48": 1562890752.0, - "49": 1562391552.0, - "50": 1562682368.0 + "1": 1557108224.0, + "2": 1556795392.0, + "3": 1556089856.0, + "4": 1555959296.0, + "5": 1555885568.0, + "6": 1556037632.0, + "7": 1556025344.0, + "8": 1556239872.0, + "9": 1557112832.0, + "10": 1555885568.0, + "11": 1556637696.0, + "12": 1556511232.0, + "13": 1557992448.0, + "14": 1556232192.0, + "15": 1557710848.0, + "16": 1556088320.0, + "17": 1558993408.0, + "18": 1558411264.0, + "19": 1556663296.0, + "20": 1556557312.0, + "21": 1556139008.0, + "22": 1556088320.0, + "23": 1556240384.0, + "24": 1557671424.0, + "25": 1556866048.0, + "26": 1557909504.0, + "27": 1556139008.0, + "28": 1558253056.0, + "29": 1557016064.0, + "30": 1557436416.0, + "31": 1557824512.0, + "32": 1558845952.0, + "33": 1556866048.0, + "34": 1556291072.0, + "35": 1556967424.0, + "36": 1557195264.0, + "37": 1557520384.0, + "38": 1559213056.0, + "39": 1558776832.0, + "40": 1556592128.0, + "41": 1557733888.0, + "42": 1557342208.0, + "43": 1556798464.0, + "44": 1558247936.0, + "45": 1558446080.0, + "46": 1557450240.0, + "47": 1558625280.0, + "48": 1557008896.0, + "49": 1556747264.0, + "50": 1558415360.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3488102400.0, - "2": 4062251520.0, - "3": 4062251520.0, - "4": 4062251520.0, - "5": 4062251520.0, - "6": 4068555776.0, - "7": 4068555776.0, - "8": 4073248256.0, - "9": 4073248256.0, - "10": 4073248256.0, - "11": 4073248256.0, - "12": 4073248256.0, - "13": 4073248256.0, - "14": 4073248256.0, - "15": 4073248256.0, - "16": 4073248256.0, - "17": 4073248256.0, - "18": 4073248256.0, - "19": 4073248256.0, - "20": 4073248256.0, - "21": 4073248256.0, - "22": 4073248256.0, - "23": 4073248256.0, - "24": 4073248256.0, - "25": 4073248256.0, - "26": 4073248256.0, - "27": 4073248256.0, - "28": 4073248256.0, - "29": 4073248256.0, - "30": 4073248256.0, - "31": 4073248256.0, - "32": 4073248256.0, - "33": 4073248256.0, - "34": 4073248256.0, - "35": 4073248256.0, - "36": 4073248256.0, - "37": 4073248256.0, - "38": 4073248256.0, - "39": 4073248256.0, - "40": 4073248256.0, - "41": 4073248256.0, - "42": 4073248256.0, - "43": 4073248256.0, - "44": 4073248256.0, - "45": 4073248256.0, - "46": 4073248256.0, - "47": 4073248256.0, - "48": 4073248256.0, - "49": 4073248256.0, - "50": 4073248256.0 + "1": 3490111488.0, + "2": 4057019904.0, + "3": 4057019904.0, + "4": 4057019904.0, + "5": 4057019904.0, + "6": 4064487424.0, + "7": 4064487424.0, + "8": 4067570176.0, + "9": 4067570176.0, + "10": 4067570176.0, + "11": 4067570176.0, + "12": 4067570176.0, + "13": 4067570176.0, + "14": 4067570176.0, + "15": 4067570176.0, + "16": 4067570176.0, + "17": 4067570176.0, + "18": 4067570176.0, + "19": 4067570176.0, + "20": 4067570176.0, + "21": 4067570176.0, + "22": 4067570176.0, + "23": 4067570176.0, + "24": 4067570176.0, + "25": 4067570176.0, + "26": 4067570176.0, + "27": 4067570176.0, + "28": 4067570176.0, + "29": 4067570176.0, + "30": 4067570176.0, + "31": 4067570176.0, + "32": 4067570176.0, + "33": 4067570176.0, + "34": 4067570176.0, + "35": 4067570176.0, + "36": 4067570176.0, + "37": 4067570176.0, + "38": 4067570176.0, + "39": 4067570176.0, + "40": 4067570176.0, + "41": 4067570176.0, + "42": 4067570176.0, + "43": 4067570176.0, + "44": 4067570176.0, + "45": 4067570176.0, + "46": 4067570176.0, + "47": 4067570176.0, + "48": 4067570176.0, + "49": 4067570176.0, + "50": 4067570176.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.79622, - "3": 1.21007, - "4": 1.14826, - "5": 0.83739, - "6": 0.99678, - "7": 1.06431, - "8": 1.46855, - "9": 1.45913, - "10": 0.92306, - "11": 1.2195, - "12": 1.29215, - "13": 0.85213, - "14": 1.5355, - "15": 0.84514, - "16": 1.16531, - "17": 1.04874, - "18": 1.20965, - "19": 1.05465, - "20": 1.12464, - "21": 1.14642, - "22": 1.08156, - "23": 1.09043, - "24": 1.72097, - "25": 0.8303, - "26": 1.08398, - "27": 1.03852, - "28": 0.99051, - "29": 1.08482, - "30": 0.88763, - "31": 0.84441, - "32": 1.78145, - "33": 1.01149, - "34": 0.84605, - "35": 1.10834, - "36": 1.11263, - "37": 0.94746, - "38": 0.83858, - "39": 1.1962, - "40": 1.45473, - "41": 1.0538, - "42": 1.10087, - "43": 1.00773, - "44": 1.28551, - "45": 0.94447, - "46": 1.09943, - "47": 0.95418, - "48": 1.16128, - "49": 0.98063, - "50": 1.10117 + "2": 6.73676, + "3": 1.32569, + "4": 1.06315, + "5": 0.70209, + "6": 1.20606, + "7": 1.0589, + "8": 1.43337, + "9": 1.4526, + "10": 0.97009, + "11": 1.06936, + "12": 1.22291, + "13": 0.67281, + "14": 1.65157, + "15": 0.77725, + "16": 1.09734, + "17": 1.12045, + "18": 1.14861, + "19": 0.98737, + "20": 1.05971, + "21": 1.12329, + "22": 1.10793, + "23": 1.04343, + "24": 1.64775, + "25": 0.97789, + "26": 0.80057, + "27": 0.95454, + "28": 1.11083, + "29": 1.24731, + "30": 0.84152, + "31": 0.79535, + "32": 1.69014, + "33": 1.01162, + "34": 0.69956, + "35": 1.12457, + "36": 1.16925, + "37": 1.0859, + "38": 0.70037, + "39": 1.31349, + "40": 1.47757, + "41": 1.08169, + "42": 1.12541, + "43": 0.93423, + "44": 1.3356, + "45": 0.83074, + "46": 1.05918, + "47": 1.1368, + "48": 1.14226, + "49": 0.96214, + "50": 1.27686 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json index 4c8fb5d696b..78c5092db1e 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90655, - "4": 10.90577, - "5": 10.91353, - "6": 10.8974, - "7": 10.89795, - "8": 10.91315, - "9": 10.89647, - "10": 10.90362, - "11": 10.90058, - "12": 10.91863, - "13": 10.88451, - "14": 10.87468, - "15": 10.90997, - "16": 10.90304, - "17": 10.88702, - "18": 10.89383, - "19": 10.89477, - "20": 10.89108, - "21": 10.88732, - "22": 10.90287, - "23": 10.91197, - "24": 10.88563, - "25": 10.90095, - "26": 10.88886, - "27": 10.89671, - "28": 10.88189, - "29": 10.88993, - "30": 10.91034, - "31": 10.89135, - "32": 10.8888, - "33": 10.8978, - "34": 10.87277, - "35": 10.89612, - "36": 10.90758, - "37": 10.8709, - "38": 10.87877, - "39": 10.88567, - "40": 10.89273, - "41": 10.88396, - "42": 10.89532, - "43": 10.87994, - "44": 10.88499, - "45": 10.88349, - "46": 10.88634, - "47": 10.88604, - "48": 10.86357, - "49": 10.87627, - "50": 10.88323, - "51": 10.8932, - "52": 10.87074, - "53": 10.85901, - "54": 10.8708, - "55": 10.86805, - "56": 10.87369, - "57": 10.84455, - "58": 10.85609, - "59": 10.84667, - "60": 10.84361, - "61": 10.86077, - "62": 10.85729, - "63": 10.86045, - "64": 10.83869, - "65": 10.82777, - "66": 10.84578, - "67": 10.8274, - "68": 10.82991, - "69": 10.81829, - "70": 10.82845, - "71": 10.82632, - "72": 10.80841, - "73": 10.80779, - "74": 10.80552, - "75": 10.81455, - "76": 10.81142, - "77": 10.80752, - "78": 10.79152, - "79": 10.80266, - "80": 10.78868, - "81": 10.79558, - "82": 10.79566, - "83": 10.78578, - "84": 10.75723, - "85": 10.76273, - "86": 10.7768, - "87": 10.79587, - "88": 10.7746, - "89": 10.77528, - "90": 10.76475, - "91": 10.74024, - "92": 10.75993, - "93": 10.7468, - "94": 10.73446, - "95": 10.75268, - "96": 10.7234, - "97": 10.71513, - "98": 10.72592, - "99": 10.74621, - "100": 10.69564 + "1": 10.90111, + "2": 10.8957, + "3": 10.90636, + "4": 10.90606, + "5": 10.91349, + "6": 10.89747, + "7": 10.89834, + "8": 10.91366, + "9": 10.89559, + "10": 10.90375, + "11": 10.90014, + "12": 10.91887, + "13": 10.88393, + "14": 10.87584, + "15": 10.91035, + "16": 10.90325, + "17": 10.88717, + "18": 10.89416, + "19": 10.89487, + "20": 10.89092, + "21": 10.88636, + "22": 10.90341, + "23": 10.91256, + "24": 10.88547, + "25": 10.90164, + "26": 10.88848, + "27": 10.89705, + "28": 10.88168, + "29": 10.88997, + "30": 10.91097, + "31": 10.89128, + "32": 10.88933, + "33": 10.89772, + "34": 10.87339, + "35": 10.89595, + "36": 10.90751, + "37": 10.87112, + "38": 10.87839, + "39": 10.88579, + "40": 10.89247, + "41": 10.88364, + "42": 10.89496, + "43": 10.88058, + "44": 10.88492, + "45": 10.88361, + "46": 10.88582, + "47": 10.88565, + "48": 10.86424, + "49": 10.8759, + "50": 10.88259, + "51": 10.89305, + "52": 10.87123, + "53": 10.85905, + "54": 10.8702, + "55": 10.86807, + "56": 10.87324, + "57": 10.84465, + "58": 10.8563, + "59": 10.84628, + "60": 10.84343, + "61": 10.8604, + "62": 10.85675, + "63": 10.86117, + "64": 10.83852, + "65": 10.82804, + "66": 10.84584, + "67": 10.82846, + "68": 10.83007, + "69": 10.81894, + "70": 10.82797, + "71": 10.82677, + "72": 10.8088, + "73": 10.80753, + "74": 10.80659, + "75": 10.81472, + "76": 10.81086, + "77": 10.80817, + "78": 10.79255, + "79": 10.80218, + "80": 10.7879, + "81": 10.79487, + "82": 10.79556, + "83": 10.78629, + "84": 10.75658, + "85": 10.76241, + "86": 10.77699, + "87": 10.79624, + "88": 10.77534, + "89": 10.77559, + "90": 10.76414, + "91": 10.74033, + "92": 10.76012, + "93": 10.74683, + "94": 10.73435, + "95": 10.75231, + "96": 10.72344, + "97": 10.71428, + "98": 10.72631, + "99": 10.74637, + "100": 10.69592 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1431.0, - "4": 1283.0, - "5": 1254.0, - "6": 1261.0, - "7": 1295.0, - "8": 1160.0, - "9": 1242.0, - "10": 1143.0, - "11": 1217.0, - "12": 1226.0, - "13": 1325.0, - "14": 1163.0, - "15": 1218.0, - "16": 1232.0, - "17": 1256.0, - "18": 1128.0, - "19": 1231.0, - "20": 1137.0, - "21": 1199.0, - "22": 1215.0, - "23": 1278.0, - "24": 1179.0, - "25": 1273.0, - "26": 1237.0, - "27": 1316.0, - "28": 1183.0, - "29": 1171.0, - "30": 1295.0, - "31": 1254.0, - "32": 1258.0, - "33": 1212.0, - "34": 1220.0, - "35": 1350.0, - "36": 1214.0, - "37": 1213.0, - "38": 1259.0, - "39": 1382.0, - "40": 1357.0, - "41": 1252.0, - "42": 1115.0, - "43": 1180.0, - "44": 1237.0, - "45": 1170.0, - "46": 1438.0, - "47": 1193.0, - "48": 1124.0, - "49": 1338.0, - "50": 1377.0, - "51": 1277.0, - "52": 1234.0, - "53": 1204.0, - "54": 1132.0, - "55": 1138.0, - "56": 1147.0, - "57": 1127.0, - "58": 1290.0, - "59": 1074.0, - "60": 1286.0, - "61": 1259.0, - "62": 1179.0, - "63": 1243.0, - "64": 1322.0, - "65": 1162.0, - "66": 1182.0, - "67": 1248.0, - "68": 1276.0, - "69": 1154.0, - "70": 1173.0, - "71": 1122.0, - "72": 1236.0, - "73": 1199.0, - "74": 1354.0, - "75": 1322.0, - "76": 1301.0, - "77": 1168.0, - "78": 1294.0, - "79": 1295.0, - "80": 1216.0, - "81": 1305.0, - "82": 1201.0, - "83": 1196.0, - "84": 1150.0, - "85": 1246.0, - "86": 1309.0, - "87": 1133.0, - "88": 1166.0, - "89": 1146.0, - "90": 1389.0, - "91": 1246.0, - "92": 1195.0, - "93": 1286.0, - "94": 1318.0, - "95": 1040.0, - "96": 1248.0, - "97": 1192.0, - "98": 1376.0, - "99": 1170.0, - "100": 1292.0 + "1": 1133.0, + "2": 1120.0, + "3": 1371.0, + "4": 1297.0, + "5": 1158.0, + "6": 1283.0, + "7": 1334.0, + "8": 1122.0, + "9": 1269.0, + "10": 1134.0, + "11": 1184.0, + "12": 1223.0, + "13": 1340.0, + "14": 1232.0, + "15": 1234.0, + "16": 1179.0, + "17": 1212.0, + "18": 1083.0, + "19": 1187.0, + "20": 1174.0, + "21": 1196.0, + "22": 1227.0, + "23": 1318.0, + "24": 1113.0, + "25": 1191.0, + "26": 1243.0, + "27": 1343.0, + "28": 1251.0, + "29": 1186.0, + "30": 1221.0, + "31": 1205.0, + "32": 1269.0, + "33": 1254.0, + "34": 1196.0, + "35": 1308.0, + "36": 1277.0, + "37": 1196.0, + "38": 1321.0, + "39": 1407.0, + "40": 1299.0, + "41": 1284.0, + "42": 1170.0, + "43": 1170.0, + "44": 1252.0, + "45": 1154.0, + "46": 1403.0, + "47": 1148.0, + "48": 1176.0, + "49": 1389.0, + "50": 1329.0, + "51": 1209.0, + "52": 1224.0, + "53": 1276.0, + "54": 1198.0, + "55": 1179.0, + "56": 1166.0, + "57": 1168.0, + "58": 1318.0, + "59": 1047.0, + "60": 1118.0, + "61": 1192.0, + "62": 1108.0, + "63": 1225.0, + "64": 1256.0, + "65": 1221.0, + "66": 1228.0, + "67": 1193.0, + "68": 1279.0, + "69": 1141.0, + "70": 1248.0, + "71": 1165.0, + "72": 1230.0, + "73": 1236.0, + "74": 1336.0, + "75": 1346.0, + "76": 1259.0, + "77": 1205.0, + "78": 1217.0, + "79": 1298.0, + "80": 1228.0, + "81": 1335.0, + "82": 1198.0, + "83": 1246.0, + "84": 1152.0, + "85": 1250.0, + "86": 1287.0, + "87": 1111.0, + "88": 1169.0, + "89": 1119.0, + "90": 1402.0, + "91": 1297.0, + "92": 1194.0, + "93": 1271.0, + "94": 1409.0, + "95": 980.0, + "96": 1170.0, + "97": 1212.0, + "98": 1325.0, + "99": 1125.0, + "100": 1206.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 994062848.0, - "2": 994053120.0, + "1": 994063360.0, + "2": 994054144.0, "3": 994029056.0, - "4": 994009600.0, - "5": 994060288.0, - "6": 994043904.0, - "7": 993970688.0, - "8": 994005504.0, - "9": 994008576.0, - "10": 994040832.0, - "11": 994017280.0, - "12": 994023936.0, - "13": 994028032.0, - "14": 994029568.0, - "15": 994005504.0, + "4": 994010624.0, + "5": 994059776.0, + "6": 994042368.0, + "7": 993970176.0, + "8": 994006528.0, + "9": 994008064.0, + "10": 994039808.0, + "11": 994015232.0, + "12": 994024960.0, + "13": 994029056.0, + "14": 994030592.0, + "15": 994007040.0, "16": 993992704.0, "17": 994001408.0, - "18": 994002944.0, - "19": 994032128.0, - "20": 993999360.0, - "21": 994050560.0, - "22": 994078720.0, - "23": 994051584.0, - "24": 993963520.0, - "25": 994044928.0, - "26": 994075648.0, - "27": 994029056.0, - "28": 993983488.0, - "29": 994057216.0, - "30": 993989632.0, - "31": 993997312.0, - "32": 994005504.0, - "33": 994009088.0, - "34": 993986560.0, - "35": 993973248.0, - "36": 994038272.0, + "18": 994003968.0, + "19": 994032640.0, + "20": 993997312.0, + "21": 994052608.0, + "22": 994075648.0, + "23": 994049024.0, + "24": 993964032.0, + "25": 994045440.0, + "26": 994076672.0, + "27": 994028544.0, + "28": 993984512.0, + "29": 994056704.0, + "30": 993990656.0, + "31": 993996800.0, + "32": 994003456.0, + "33": 994011136.0, + "34": 993989120.0, + "35": 993974784.0, + "36": 994040832.0, "37": 993988608.0, - "38": 994048000.0, - "39": 994056192.0, - "40": 994000896.0, - "41": 994023424.0, - "42": 993964032.0, - "43": 994031104.0, - "44": 994015232.0, - "45": 993988096.0, - "46": 994054144.0, - "47": 994010624.0, + "38": 994046976.0, + "39": 994054144.0, + "40": 994001408.0, + "41": 994021888.0, + "42": 993964544.0, + "43": 994032640.0, + "44": 994014208.0, + "45": 993989120.0, + "46": 994052096.0, + "47": 994011136.0, "48": 994040320.0, - "49": 993998336.0, - "50": 994048000.0, - "51": 994031104.0, - "52": 994000896.0, - "53": 993953792.0, - "54": 994048512.0, - "55": 994041344.0, - "56": 994039296.0, - "57": 994099200.0, - "58": 994036224.0, - "59": 994042880.0, - "60": 994017792.0, - "61": 993987584.0, - "62": 993977344.0, - "63": 993988096.0, - "64": 994016768.0, - "65": 994022400.0, - "66": 994036736.0, - "67": 994019328.0, + "49": 993998848.0, + "50": 994044928.0, + "51": 994030592.0, + "52": 994003968.0, + "53": 993953280.0, + "54": 994051072.0, + "55": 994039808.0, + "56": 994038784.0, + "57": 994098688.0, + "58": 994035200.0, + "59": 994040832.0, + "60": 994019328.0, + "61": 993988608.0, + "62": 993980928.0, + "63": 993987584.0, + "64": 994015744.0, + "65": 994023424.0, + "66": 994037248.0, + "67": 994018816.0, "68": 994020352.0, - "69": 993982464.0, - "70": 994070016.0, - "71": 994048512.0, - "72": 994044928.0, - "73": 993976320.0, - "74": 994013184.0, - "75": 994061824.0, - "76": 994019328.0, - "77": 994062848.0, - "78": 994003968.0, - "79": 994072576.0, + "69": 993985024.0, + "70": 994067456.0, + "71": 994047488.0, + "72": 994044416.0, + "73": 993973760.0, + "74": 994015744.0, + "75": 994060800.0, + "76": 994017792.0, + "77": 994060288.0, + "78": 994006016.0, + "79": 994072064.0, "80": 993993728.0, - "81": 994045952.0, - "82": 993991680.0, - "83": 994055680.0, - "84": 994026496.0, - "85": 994033664.0, - "86": 994017280.0, + "81": 994046464.0, + "82": 993994752.0, + "83": 994054656.0, + "84": 994024448.0, + "85": 994032128.0, + "86": 994016256.0, "87": 994030080.0, - "88": 994014208.0, - "89": 994002432.0, - "90": 994024448.0, - "91": 994012160.0, - "92": 994010624.0, - "93": 994039808.0, - "94": 994013184.0, - "95": 994034688.0, - "96": 994028032.0, - "97": 993995776.0, - "98": 994034176.0, - "99": 994049024.0, - "100": 994081280.0 + "88": 994010112.0, + "89": 994001408.0, + "90": 994024960.0, + "91": 994011136.0, + "92": 994009600.0, + "93": 994040320.0, + "94": 994015232.0, + "95": 994036736.0, + "96": 994028544.0, + "97": 993997312.0, + "98": 994035712.0, + "99": 994051584.0, + "100": 994083328.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3207447552.0, - "2": 3491065344.0, - "3": 3491065344.0, - "4": 3491065344.0, - "5": 3497349632.0, - "6": 3497349632.0, - "7": 3497349632.0, - "8": 3497349632.0, - "9": 3497349632.0, - "10": 3497349632.0, - "11": 3497349632.0, - "12": 3497349632.0, - "13": 3497349632.0, - "14": 3497349632.0, - "15": 3497349632.0, - "16": 3497349632.0, - "17": 3497349632.0, - "18": 3497349632.0, - "19": 3497349632.0, - "20": 3497349632.0, - "21": 3497349632.0, - "22": 3506985472.0, - "23": 3506985472.0, - "24": 3506985472.0, - "25": 3506985472.0, - "26": 3512014336.0, - "27": 3512014336.0, - "28": 3512014336.0, - "29": 3512014336.0, - "30": 3512014336.0, - "31": 3512014336.0, - "32": 3512014336.0, - "33": 3512014336.0, - "34": 3512014336.0, - "35": 3512014336.0, - "36": 3512014336.0, - "37": 3512014336.0, - "38": 3512014336.0, - "39": 3512014336.0, - "40": 3512014336.0, - "41": 3512014336.0, - "42": 3512014336.0, - "43": 3512014336.0, - "44": 3512014336.0, - "45": 3512014336.0, - "46": 3512014336.0, - "47": 3512014336.0, - "48": 3512014336.0, - "49": 3512014336.0, - "50": 3512014336.0, - "51": 3512014336.0, - "52": 3512014336.0, - "53": 3512014336.0, - "54": 3512014336.0, - "55": 3512014336.0, - "56": 3512014336.0, - "57": 3526792704.0, - "58": 3526792704.0, - "59": 3526792704.0, - "60": 3526792704.0, - "61": 3526792704.0, - "62": 3526792704.0, - "63": 3526792704.0, - "64": 3526792704.0, - "65": 3526792704.0, - "66": 3526792704.0, - "67": 3526792704.0, - "68": 3526792704.0, - "69": 3526792704.0, - "70": 3526792704.0, - "71": 3526792704.0, - "72": 3526792704.0, - "73": 3526792704.0, - "74": 3526792704.0, - "75": 3526792704.0, - "76": 3526792704.0, - "77": 3526792704.0, - "78": 3526792704.0, - "79": 3526792704.0, - "80": 3526792704.0, - "81": 3526792704.0, - "82": 3526792704.0, - "83": 3526792704.0, - "84": 3526792704.0, - "85": 3526792704.0, - "86": 3526792704.0, - "87": 3526792704.0, - "88": 3526792704.0, - "89": 3526792704.0, - "90": 3526792704.0, - "91": 3526792704.0, - "92": 3526792704.0, - "93": 3526792704.0, - "94": 3526792704.0, - "95": 3526792704.0, - "96": 3526792704.0, - "97": 3526792704.0, - "98": 3526792704.0, - "99": 3526792704.0, - "100": 3526792704.0 + "1": 3205999104.0, + "2": 3490994688.0, + "3": 3490994688.0, + "4": 3490994688.0, + "5": 3498078208.0, + "6": 3498078208.0, + "7": 3498078208.0, + "8": 3498078208.0, + "9": 3498078208.0, + "10": 3498078208.0, + "11": 3498078208.0, + "12": 3498078208.0, + "13": 3498078208.0, + "14": 3498078208.0, + "15": 3498078208.0, + "16": 3498078208.0, + "17": 3498078208.0, + "18": 3498078208.0, + "19": 3498078208.0, + "20": 3498078208.0, + "21": 3498078208.0, + "22": 3507077632.0, + "23": 3507077632.0, + "24": 3507077632.0, + "25": 3507077632.0, + "26": 3510859264.0, + "27": 3510859264.0, + "28": 3510859264.0, + "29": 3510859264.0, + "30": 3510859264.0, + "31": 3510859264.0, + "32": 3510859264.0, + "33": 3510859264.0, + "34": 3510859264.0, + "35": 3510859264.0, + "36": 3510859264.0, + "37": 3510859264.0, + "38": 3510859264.0, + "39": 3510859264.0, + "40": 3510859264.0, + "41": 3510859264.0, + "42": 3510859264.0, + "43": 3510859264.0, + "44": 3510859264.0, + "45": 3510859264.0, + "46": 3510859264.0, + "47": 3510859264.0, + "48": 3510859264.0, + "49": 3510859264.0, + "50": 3510859264.0, + "51": 3510859264.0, + "52": 3510859264.0, + "53": 3510859264.0, + "54": 3510859264.0, + "55": 3510859264.0, + "56": 3510859264.0, + "57": 3526288384.0, + "58": 3526288384.0, + "59": 3526288384.0, + "60": 3526288384.0, + "61": 3526288384.0, + "62": 3526288384.0, + "63": 3526288384.0, + "64": 3526288384.0, + "65": 3526288384.0, + "66": 3526288384.0, + "67": 3526288384.0, + "68": 3526288384.0, + "69": 3526288384.0, + "70": 3526288384.0, + "71": 3526288384.0, + "72": 3526288384.0, + "73": 3526288384.0, + "74": 3526288384.0, + "75": 3526288384.0, + "76": 3526288384.0, + "77": 3526288384.0, + "78": 3526288384.0, + "79": 3526288384.0, + "80": 3526288384.0, + "81": 3526288384.0, + "82": 3526288384.0, + "83": 3526288384.0, + "84": 3526288384.0, + "85": 3526288384.0, + "86": 3526288384.0, + "87": 3526288384.0, + "88": 3526288384.0, + "89": 3526288384.0, + "90": 3526288384.0, + "91": 3526288384.0, + "92": 3526288384.0, + "93": 3526288384.0, + "94": 3526288384.0, + "95": 3526288384.0, + "96": 3526288384.0, + "97": 3526288384.0, + "98": 3526288384.0, + "99": 3526288384.0, + "100": 3526288384.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.43392, - "3": 0.22954, - "4": 0.2187, - "5": 0.21296, - "6": 0.20548, - "7": 0.20909, - "8": 0.22033, - "9": 0.20266, - "10": 0.20368, - "11": 0.20814, - "12": 0.19574, - "13": 0.19885, - "14": 0.19615, - "15": 0.20178, - "16": 0.19024, - "17": 0.19778, - "18": 0.19123, - "19": 0.19898, - "20": 0.19976, - "21": 0.18857, - "22": 0.19138, - "23": 0.19297, - "24": 0.19637, - "25": 0.19458, - "26": 0.19364, - "27": 0.1858, - "28": 0.17947, - "29": 0.19539, - "30": 0.19752, - "31": 0.18516, - "32": 0.19246, - "33": 0.19096, - "34": 0.2006, - "35": 0.19289, - "36": 0.18703, - "37": 0.20608, - "38": 0.18975, - "39": 0.18731, - "40": 0.18631, - "41": 0.18456, - "42": 0.18662, - "43": 0.18589, - "44": 0.18125, - "45": 0.19308, - "46": 0.17989, - "47": 0.1867, - "48": 0.18404, - "49": 0.18095, - "50": 0.18971, - "51": 0.25875, - "52": 0.20921, - "53": 0.19814, - "54": 0.19506, - "55": 0.17801, - "56": 0.17839, - "57": 0.18558, - "58": 0.1838, - "59": 0.17996, - "60": 0.18455, - "61": 0.19164, - "62": 0.1906, - "63": 0.17811, - "64": 0.18037, - "65": 0.18733, - "66": 0.18942, - "67": 0.18393, - "68": 0.18292, - "69": 0.18248, - "70": 0.1847, - "71": 0.17859, - "72": 0.18218, - "73": 0.18167, - "74": 0.177, - "75": 0.18191, - "76": 0.18191, - "77": 0.18558, - "78": 0.17902, - "79": 0.18474, - "80": 0.18875, - "81": 0.19061, - "82": 0.20019, - "83": 0.1804, - "84": 0.19066, - "85": 0.18001, - "86": 0.18562, - "87": 0.1847, - "88": 0.18147, - "89": 0.19573, - "90": 0.17983, - "91": 0.18358, - "92": 0.17809, - "93": 0.17549, - "94": 0.18735, - "95": 0.18122, - "96": 0.18269, - "97": 0.18541, - "98": 0.1788, - "99": 0.18398, - "100": 0.19234 + "2": 4.80343, + "3": 0.24461, + "4": 0.21905, + "5": 0.23456, + "6": 0.22151, + "7": 0.22611, + "8": 0.2404, + "9": 0.21788, + "10": 0.22319, + "11": 0.22849, + "12": 0.21807, + "13": 0.2241, + "14": 0.21328, + "15": 0.21924, + "16": 0.19766, + "17": 0.2119, + "18": 0.20845, + "19": 0.2102, + "20": 0.21676, + "21": 0.2071, + "22": 0.21066, + "23": 0.20145, + "24": 0.20861, + "25": 0.22351, + "26": 0.20212, + "27": 0.19701, + "28": 0.19726, + "29": 0.21428, + "30": 0.20896, + "31": 0.20412, + "32": 0.21173, + "33": 0.20281, + "34": 0.20432, + "35": 0.20437, + "36": 0.20527, + "37": 0.20735, + "38": 0.19734, + "39": 0.19343, + "40": 0.20184, + "41": 0.19924, + "42": 0.19429, + "43": 0.20453, + "44": 0.19924, + "45": 0.20236, + "46": 0.19519, + "47": 0.1998, + "48": 0.20296, + "49": 0.20318, + "50": 0.19638, + "51": 0.25423, + "52": 0.22224, + "53": 0.20083, + "54": 0.19378, + "55": 0.19638, + "56": 0.19433, + "57": 0.21583, + "58": 0.20227, + "59": 0.19823, + "60": 0.2012, + "61": 0.20156, + "62": 0.21045, + "63": 0.19778, + "64": 0.19188, + "65": 0.19114, + "66": 0.19429, + "67": 0.21388, + "68": 0.19833, + "69": 0.20586, + "70": 0.19598, + "71": 0.1923, + "72": 0.19411, + "73": 0.20635, + "74": 0.20087, + "75": 0.19463, + "76": 0.19501, + "77": 0.20497, + "78": 0.19948, + "79": 0.19854, + "80": 0.19801, + "81": 0.21063, + "82": 0.20059, + "83": 0.19918, + "84": 0.20511, + "85": 0.19701, + "86": 0.20162, + "87": 0.20013, + "88": 0.20349, + "89": 0.19494, + "90": 0.19192, + "91": 0.20582, + "92": 0.19576, + "93": 0.20142, + "94": 0.20002, + "95": 0.1982, + "96": 0.20164, + "97": 0.20778, + "98": 0.19874, + "99": 0.19422, + "100": 0.20224 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json index bac266b9eef..7edbab4f74a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.9252, - "4": 10.91732, - "5": 10.92716, - "6": 10.92029, - "7": 10.91669, - "8": 10.91796, - "9": 10.93887, - "10": 10.92403, - "11": 10.91598, + "1": 10.93064, + "2": 10.92264, + "3": 10.92451, + "4": 10.91716, + "5": 10.92714, + "6": 10.92064, + "7": 10.91647, + "8": 10.91788, + "9": 10.93809, + "10": 10.924, + "11": 10.91584, "12": 10.92558, - "13": 10.93454, - "14": 10.9158, - "15": 10.92838, - "16": 10.91826, - "17": 10.9274, - "18": 10.91269, - "19": 10.93069, - "20": 10.90057, - "21": 10.93975, - "22": 10.92531, - "23": 10.92976, - "24": 10.90567, - "25": 10.91494, - "26": 10.92515, - "27": 10.92786, - "28": 10.92017, - "29": 10.92534, - "30": 10.91409, - "31": 10.91023, - "32": 10.91855, - "33": 10.92026, - "34": 10.90054, - "35": 10.91897, - "36": 10.90741, - "37": 10.91267, - "38": 10.92379, - "39": 10.91251, - "40": 10.91043, - "41": 10.91309, - "42": 10.90891, - "43": 10.90503, - "44": 10.90748, - "45": 10.89756, - "46": 10.90681, - "47": 10.90215, - "48": 10.88414, - "49": 10.90022, - "50": 10.89863, - "51": 10.89746, - "52": 10.90395, - "53": 10.90118, - "54": 10.88728, - "55": 10.8869, - "56": 10.88881, - "57": 10.88731, - "58": 10.8888, - "59": 10.88231, - "60": 10.87378, - "61": 10.8731, - "62": 10.8684, - "63": 10.87569, - "64": 10.85934, - "65": 10.86089, - "66": 10.85671, - "67": 10.86152, - "68": 10.86019, - "69": 10.85018, - "70": 10.86781, - "71": 10.84867, - "72": 10.8405, - "73": 10.848, - "74": 10.83857, - "75": 10.83617, - "76": 10.83457, - "77": 10.82953, - "78": 10.82324, - "79": 10.82899, - "80": 10.82879, - "81": 10.81798, - "82": 10.81714, - "83": 10.80413, - "84": 10.78245, - "85": 10.78302, - "86": 10.80172, - "87": 10.79579, - "88": 10.79542, - "89": 10.77659, - "90": 10.78278, - "91": 10.79029, - "92": 10.78291, - "93": 10.75496, - "94": 10.76061, - "95": 10.77271, - "96": 10.73624, - "97": 10.74137, - "98": 10.74745, - "99": 10.76246, - "100": 10.74181 + "13": 10.9341, + "14": 10.91552, + "15": 10.92865, + "16": 10.91861, + "17": 10.92663, + "18": 10.91342, + "19": 10.93055, + "20": 10.89988, + "21": 10.94059, + "22": 10.92549, + "23": 10.92966, + "24": 10.90596, + "25": 10.91497, + "26": 10.92625, + "27": 10.92807, + "28": 10.92038, + "29": 10.92592, + "30": 10.91434, + "31": 10.91031, + "32": 10.919, + "33": 10.92014, + "34": 10.89998, + "35": 10.91944, + "36": 10.90693, + "37": 10.91287, + "38": 10.92352, + "39": 10.91196, + "40": 10.91094, + "41": 10.91323, + "42": 10.90919, + "43": 10.90589, + "44": 10.90761, + "45": 10.89741, + "46": 10.90636, + "47": 10.90203, + "48": 10.88495, + "49": 10.90015, + "50": 10.89905, + "51": 10.89814, + "52": 10.90393, + "53": 10.90158, + "54": 10.88808, + "55": 10.88643, + "56": 10.88868, + "57": 10.88748, + "58": 10.88859, + "59": 10.88263, + "60": 10.87391, + "61": 10.87367, + "62": 10.86781, + "63": 10.87607, + "64": 10.85882, + "65": 10.86026, + "66": 10.85712, + "67": 10.86113, + "68": 10.86032, + "69": 10.84979, + "70": 10.86738, + "71": 10.8492, + "72": 10.84019, + "73": 10.84868, + "74": 10.83869, + "75": 10.83621, + "76": 10.83434, + "77": 10.82938, + "78": 10.82319, + "79": 10.82846, + "80": 10.82987, + "81": 10.81761, + "82": 10.81704, + "83": 10.80437, + "84": 10.78281, + "85": 10.78319, + "86": 10.80029, + "87": 10.7956, + "88": 10.79539, + "89": 10.77706, + "90": 10.78274, + "91": 10.79015, + "92": 10.78298, + "93": 10.75507, + "94": 10.7607, + "95": 10.77273, + "96": 10.7355, + "97": 10.7421, + "98": 10.74705, + "99": 10.76264, + "100": 10.74262 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 49241.0, - "4": 48921.0, - "5": 50418.0, - "6": 49904.0, - "7": 51727.0, - "8": 49428.0, - "9": 50948.0, - "10": 52087.0, - "11": 50232.0, - "12": 48851.0, - "13": 52086.0, - "14": 50015.0, - "15": 47551.0, - "16": 49640.0, - "17": 49732.0, - "18": 51675.0, - "19": 50978.0, - "20": 52800.0, - "21": 49547.0, - "22": 49732.0, - "23": 53396.0, - "24": 51250.0, - "25": 48900.0, - "26": 52786.0, - "27": 51903.0, - "28": 48333.0, - "29": 49997.0, - "30": 51256.0, - "31": 52920.0, - "32": 49183.0, - "33": 52496.0, - "34": 50444.0, - "35": 50976.0, - "36": 50903.0, - "37": 49604.0, - "38": 51329.0, - "39": 50700.0, - "40": 50997.0, - "41": 48307.0, - "42": 50284.0, - "43": 51039.0, - "44": 47888.0, - "45": 54059.0, - "46": 47726.0, - "47": 50619.0, - "48": 50261.0, - "49": 55116.0, - "50": 50618.0, - "51": 49136.0, - "52": 50826.0, - "53": 49526.0, - "54": 49051.0, - "55": 48764.0, - "56": 48419.0, - "57": 48283.0, - "58": 52488.0, - "59": 50348.0, - "60": 49754.0, - "61": 47059.0, - "62": 50826.0, - "63": 48461.0, - "64": 55874.0, - "65": 51256.0, - "66": 48951.0, - "67": 51893.0, - "68": 49259.0, - "69": 54945.0, - "70": 50024.0, - "71": 48846.0, - "72": 51735.0, - "73": 52873.0, - "74": 49777.0, - "75": 50232.0, - "76": 51309.0, - "77": 50257.0, - "78": 49979.0, - "79": 48610.0, - "80": 51996.0, - "81": 50042.0, - "82": 47216.0, - "83": 52389.0, - "84": 48624.0, - "85": 50422.0, - "86": 49534.0, - "87": 46782.0, - "88": 48381.0, - "89": 49533.0, - "90": 52461.0, - "91": 50240.0, - "92": 50070.0, - "93": 50832.0, - "94": 51690.0, - "95": 48408.0, - "96": 49903.0, - "97": 49691.0, - "98": 48279.0, - "99": 48860.0, - "100": 47315.0 + "1": 49180.0, + "2": 50476.0, + "3": 49283.0, + "4": 48797.0, + "5": 50222.0, + "6": 50071.0, + "7": 51516.0, + "8": 49687.0, + "9": 50960.0, + "10": 52404.0, + "11": 50454.0, + "12": 48350.0, + "13": 52171.0, + "14": 49951.0, + "15": 48080.0, + "16": 49110.0, + "17": 50288.0, + "18": 52118.0, + "19": 51099.0, + "20": 52646.0, + "21": 49816.0, + "22": 49955.0, + "23": 53512.0, + "24": 51380.0, + "25": 48547.0, + "26": 52530.0, + "27": 51993.0, + "28": 48540.0, + "29": 50032.0, + "30": 51384.0, + "31": 53098.0, + "32": 49324.0, + "33": 51925.0, + "34": 50079.0, + "35": 51371.0, + "36": 51207.0, + "37": 49665.0, + "38": 50936.0, + "39": 50632.0, + "40": 50968.0, + "41": 48043.0, + "42": 50241.0, + "43": 50548.0, + "44": 47518.0, + "45": 54359.0, + "46": 47844.0, + "47": 50697.0, + "48": 50607.0, + "49": 54927.0, + "50": 51129.0, + "51": 48773.0, + "52": 50817.0, + "53": 49252.0, + "54": 48794.0, + "55": 49077.0, + "56": 48641.0, + "57": 48141.0, + "58": 52709.0, + "59": 50075.0, + "60": 49760.0, + "61": 47761.0, + "62": 50557.0, + "63": 48459.0, + "64": 56353.0, + "65": 51179.0, + "66": 48545.0, + "67": 51570.0, + "68": 49653.0, + "69": 54155.0, + "70": 49808.0, + "71": 49534.0, + "72": 52109.0, + "73": 52558.0, + "74": 49904.0, + "75": 50500.0, + "76": 52020.0, + "77": 49771.0, + "78": 49836.0, + "79": 48741.0, + "80": 52069.0, + "81": 50079.0, + "82": 46782.0, + "83": 52982.0, + "84": 48744.0, + "85": 50660.0, + "86": 50050.0, + "87": 46875.0, + "88": 48017.0, + "89": 48621.0, + "90": 51995.0, + "91": 50317.0, + "92": 50374.0, + "93": 50702.0, + "94": 51589.0, + "95": 48646.0, + "96": 50031.0, + "97": 49732.0, + "98": 48573.0, + "99": 48599.0, + "100": 47055.0 } }, "mem-allocated-bytes": { @@ -218,105 +218,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 870734336.0, - "2": 870755840.0, + "1": 870735872.0, + "2": 870754816.0, "3": 870738944.0, - "4": 870736384.0, - "5": 870749184.0, - "6": 870733824.0, - "7": 870728704.0, - "8": 870757376.0, - "9": 870707712.0, - "10": 870706176.0, - "11": 870695424.0, - "12": 870721536.0, - "13": 870721536.0, - "14": 870710272.0, - "15": 870740992.0, - "16": 870730240.0, - "17": 870743040.0, - "18": 870720000.0, - "19": 870715392.0, - "20": 870729216.0, - "21": 870750720.0, - "22": 870684672.0, - "23": 870728192.0, + "4": 870736896.0, + "5": 870747136.0, + "6": 870734848.0, + "7": 870729728.0, + "8": 870758400.0, + "9": 870706688.0, + "10": 870708224.0, + "11": 870691840.0, + "12": 870722560.0, + "13": 870722048.0, + "14": 870712320.0, + "15": 870742016.0, + "16": 870731264.0, + "17": 870742528.0, + "18": 870718976.0, + "19": 870715904.0, + "20": 870726656.0, + "21": 870749696.0, + "22": 870685696.0, + "23": 870727168.0, "24": 870740992.0, - "25": 870703616.0, + "25": 870705664.0, "26": 870743040.0, - "27": 870746112.0, + "27": 870745600.0, "28": 870710272.0, - "29": 870728192.0, - "30": 870737920.0, - "31": 870701568.0, - "32": 870721536.0, - "33": 870708736.0, + "29": 870727680.0, + "30": 870737408.0, + "31": 870700544.0, + "32": 870722048.0, + "33": 870707712.0, "34": 870750208.0, - "35": 870679552.0, - "36": 870751744.0, + "35": 870678016.0, + "36": 870753792.0, "37": 870719488.0, - "38": 870747648.0, - "39": 870707712.0, + "38": 870744064.0, + "39": 870709760.0, "40": 870759424.0, - "41": 870723584.0, - "42": 870704128.0, - "43": 870708736.0, - "44": 870715392.0, + "41": 870726656.0, + "42": 870705664.0, + "43": 870706176.0, + "44": 870716416.0, "45": 870730752.0, - "46": 870681088.0, - "47": 870702592.0, - "48": 870694400.0, - "49": 870727680.0, - "50": 870706176.0, - "51": 870675968.0, - "52": 870722560.0, + "46": 870681600.0, + "47": 870704640.0, + "48": 870693888.0, + "49": 870726144.0, + "50": 870706688.0, + "51": 870676480.0, + "52": 870724096.0, "53": 870680576.0, - "54": 870742528.0, - "55": 870687232.0, - "56": 870735872.0, - "57": 870709248.0, - "58": 870713344.0, + "54": 870741504.0, + "55": 870686208.0, + "56": 870737920.0, + "57": 870706176.0, + "58": 870713856.0, "59": 870715904.0, - "60": 870678528.0, - "61": 870660608.0, - "62": 870694912.0, - "63": 870702592.0, - "64": 870717952.0, - "65": 870720000.0, - "66": 870642176.0, - "67": 870725632.0, - "68": 870674944.0, - "69": 870699008.0, - "70": 870664704.0, - "71": 870688768.0, - "72": 870688256.0, - "73": 870685184.0, - "74": 870651904.0, - "75": 870676480.0, - "76": 870678528.0, - "77": 870715392.0, - "78": 870711296.0, - "79": 870681088.0, - "80": 870627328.0, - "81": 870669824.0, - "82": 870642688.0, + "60": 870680576.0, + "61": 870662656.0, + "62": 870695936.0, + "63": 870699008.0, + "64": 870713344.0, + "65": 870717952.0, + "66": 870641152.0, + "67": 870729216.0, + "68": 870673920.0, + "69": 870702080.0, + "70": 870668288.0, + "71": 870687744.0, + "72": 870685184.0, + "73": 870686208.0, + "74": 870650368.0, + "75": 870676992.0, + "76": 870678016.0, + "77": 870716928.0, + "78": 870710784.0, + "79": 870678528.0, + "80": 870626816.0, + "81": 870669312.0, + "82": 870644736.0, "83": 870636032.0, - "84": 870680576.0, + "84": 870677504.0, "85": 870650880.0, - "86": 870646784.0, - "87": 870637056.0, - "88": 870647808.0, - "89": 870635520.0, - "90": 870673920.0, - "91": 870638592.0, - "92": 870637056.0, + "86": 870644736.0, + "87": 870636032.0, + "88": 870648832.0, + "89": 870634496.0, + "90": 870674432.0, + "91": 870637056.0, + "92": 870639104.0, "93": 870653952.0, - "94": 870636032.0, - "95": 870631936.0, - "96": 870638080.0, - "97": 870631936.0, + "94": 870635520.0, + "95": 870631424.0, + "96": 870637056.0, + "97": 870630912.0, "98": 870680064.0, - "99": 870618112.0, + "99": 870621184.0, "100": 870622720.0 } }, @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3339315712.0, - "2": 3489557504.0, - "3": 3489557504.0, - "4": 3489557504.0, - "5": 3489557504.0, - "6": 3489557504.0, - "7": 3489557504.0, - "8": 3489557504.0, - "9": 3489557504.0, - "10": 3489557504.0, - "11": 3489557504.0, - "12": 3489557504.0, - "13": 3489557504.0, - "14": 3489610752.0, - "15": 3489610752.0, - "16": 3489610752.0, - "17": 3489610752.0, - "18": 3489610752.0, - "19": 3489610752.0, - "20": 3489610752.0, - "21": 3489610752.0, - "22": 3489610752.0, - "23": 3489610752.0, - "24": 3489610752.0, - "25": 3489610752.0, - "26": 3489610752.0, - "27": 3489610752.0, - "28": 3489610752.0, - "29": 3489610752.0, - "30": 3489610752.0, - "31": 3489610752.0, - "32": 3489610752.0, - "33": 3489610752.0, - "34": 3489610752.0, - "35": 3489610752.0, - "36": 3489610752.0, - "37": 3489610752.0, - "38": 3489610752.0, - "39": 3489610752.0, - "40": 3489610752.0, - "41": 3489610752.0, - "42": 3489610752.0, - "43": 3489610752.0, - "44": 3489610752.0, - "45": 3489610752.0, - "46": 3489610752.0, - "47": 3489610752.0, - "48": 3489610752.0, - "49": 3489610752.0, - "50": 3489610752.0, - "51": 3489610752.0, - "52": 3489610752.0, - "53": 3489610752.0, - "54": 3489610752.0, - "55": 3489610752.0, - "56": 3489610752.0, - "57": 3489610752.0, - "58": 3489610752.0, - "59": 3489610752.0, - "60": 3489610752.0, - "61": 3489610752.0, - "62": 3489610752.0, - "63": 3489610752.0, - "64": 3489610752.0, - "65": 3489610752.0, - "66": 3489610752.0, - "67": 3489610752.0, - "68": 3489610752.0, - "69": 3489610752.0, - "70": 3489610752.0, - "71": 3489610752.0, - "72": 3489610752.0, - "73": 3489610752.0, - "74": 3489610752.0, - "75": 3489610752.0, - "76": 3489610752.0, - "77": 3489610752.0, - "78": 3489610752.0, - "79": 3489610752.0, - "80": 3489610752.0, - "81": 3489610752.0, - "82": 3489610752.0, - "83": 3489610752.0, - "84": 3489610752.0, - "85": 3489610752.0, - "86": 3489610752.0, - "87": 3489610752.0, - "88": 3489610752.0, - "89": 3489610752.0, - "90": 3489610752.0, - "91": 3489610752.0, - "92": 3489610752.0, - "93": 3489610752.0, - "94": 3489610752.0, - "95": 3489610752.0, - "96": 3489610752.0, - "97": 3489610752.0, - "98": 3489610752.0, - "99": 3489610752.0, - "100": 3489610752.0 + "1": 3338426368.0, + "2": 3490031616.0, + "3": 3490031616.0, + "4": 3490031616.0, + "5": 3490031616.0, + "6": 3490031616.0, + "7": 3490031616.0, + "8": 3490031616.0, + "9": 3490031616.0, + "10": 3490031616.0, + "11": 3490031616.0, + "12": 3490031616.0, + "13": 3490031616.0, + "14": 3490031616.0, + "15": 3490031616.0, + "16": 3490031616.0, + "17": 3490031616.0, + "18": 3490031616.0, + "19": 3490031616.0, + "20": 3490031616.0, + "21": 3490031616.0, + "22": 3490031616.0, + "23": 3490031616.0, + "24": 3490031616.0, + "25": 3490031616.0, + "26": 3490031616.0, + "27": 3490031616.0, + "28": 3490031616.0, + "29": 3490031616.0, + "30": 3490031616.0, + "31": 3490031616.0, + "32": 3490031616.0, + "33": 3490031616.0, + "34": 3490031616.0, + "35": 3490031616.0, + "36": 3490031616.0, + "37": 3490031616.0, + "38": 3490031616.0, + "39": 3490031616.0, + "40": 3490031616.0, + "41": 3490031616.0, + "42": 3490031616.0, + "43": 3490031616.0, + "44": 3490031616.0, + "45": 3490031616.0, + "46": 3490031616.0, + "47": 3490031616.0, + "48": 3490031616.0, + "49": 3490031616.0, + "50": 3490031616.0, + "51": 3490031616.0, + "52": 3490031616.0, + "53": 3490031616.0, + "54": 3490031616.0, + "55": 3490031616.0, + "56": 3490031616.0, + "57": 3490031616.0, + "58": 3490031616.0, + "59": 3490031616.0, + "60": 3490031616.0, + "61": 3490031616.0, + "62": 3490031616.0, + "63": 3490031616.0, + "64": 3490031616.0, + "65": 3490031616.0, + "66": 3490031616.0, + "67": 3490031616.0, + "68": 3490031616.0, + "69": 3490031616.0, + "70": 3490031616.0, + "71": 3490031616.0, + "72": 3490031616.0, + "73": 3490031616.0, + "74": 3490031616.0, + "75": 3490031616.0, + "76": 3490031616.0, + "77": 3490031616.0, + "78": 3490031616.0, + "79": 3490031616.0, + "80": 3490031616.0, + "81": 3490031616.0, + "82": 3490031616.0, + "83": 3490031616.0, + "84": 3490031616.0, + "85": 3490031616.0, + "86": 3490031616.0, + "87": 3490031616.0, + "88": 3490031616.0, + "89": 3490031616.0, + "90": 3490031616.0, + "91": 3490031616.0, + "92": 3490031616.0, + "93": 3490031616.0, + "94": 3490031616.0, + "95": 3490031616.0, + "96": 3490031616.0, + "97": 3490031616.0, + "98": 3490031616.0, + "99": 3490031616.0, + "100": 3490031616.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.58549, - "3": 0.45873, - "4": 0.72389, - "5": 0.43572, - "6": 0.61689, - "7": 0.71091, - "8": 0.80567, - "9": 0.53532, - "10": 0.7268, - "11": 1.09113, - "12": 0.41894, - "13": 0.61108, - "14": 0.96012, - "15": 0.42046, - "16": 1.19937, - "17": 0.68288, - "18": 1.00063, - "19": 0.42525, - "20": 0.43066, - "21": 0.90114, - "22": 0.78844, - "23": 1.14833, - "24": 0.41739, - "25": 0.50646, - "26": 0.94527, - "27": 0.42032, - "28": 0.58231, - "29": 0.88167, - "30": 0.42641, - "31": 0.41707, - "32": 0.86526, - "33": 0.54382, - "34": 1.09909, - "35": 0.40999, - "36": 0.62898, - "37": 0.56949, - "38": 0.46383, - "39": 0.62988, - "40": 0.80889, - "41": 0.69177, - "42": 0.85919, - "43": 0.54109, - "44": 0.94286, - "45": 0.52102, - "46": 0.81405, - "47": 0.68935, - "48": 1.18723, - "49": 0.46696, - "50": 0.45085, - "51": 0.48617, - "52": 0.46498, - "53": 0.43002, - "54": 0.78754, - "55": 0.77661, - "56": 0.91987, - "57": 0.43668, - "58": 0.57538, - "59": 0.8118, - "60": 0.64391, - "61": 0.42024, - "62": 0.61178, - "63": 0.59716, - "64": 0.85997, - "65": 0.51173, - "66": 0.75369, - "67": 0.60196, - "68": 0.41303, - "69": 0.51359, - "70": 1.07059, - "71": 0.40741, - "72": 0.42067, - "73": 0.57342, - "74": 0.47313, - "75": 1.01083, - "76": 0.43509, - "77": 0.43961, - "78": 1.41587, - "79": 0.40922, - "80": 0.64319, - "81": 0.71607, - "82": 0.58617, - "83": 0.81488, - "84": 0.68728, - "85": 0.41029, - "86": 0.41426, - "87": 0.68579, - "88": 0.87244, - "89": 0.45803, - "90": 0.52494, - "91": 0.50687, - "92": 0.9093, - "93": 0.44208, - "94": 0.41724, - "95": 0.74468, - "96": 0.48072, - "97": 0.53053, - "98": 0.72816, - "99": 1.40355, - "100": 0.4221 + "2": 5.31191, + "3": 0.4081, + "4": 0.75684, + "5": 0.40216, + "6": 0.79212, + "7": 0.67838, + "8": 1.14636, + "9": 0.66928, + "10": 0.39229, + "11": 1.0781, + "12": 0.3588, + "13": 0.48853, + "14": 1.01991, + "15": 0.39449, + "16": 1.00595, + "17": 0.57485, + "18": 1.08493, + "19": 0.39338, + "20": 0.52887, + "21": 0.75728, + "22": 0.76739, + "23": 0.68487, + "24": 0.3607, + "25": 0.46274, + "26": 1.16536, + "27": 0.52339, + "28": 0.61313, + "29": 0.89376, + "30": 0.35668, + "31": 0.35172, + "32": 0.98229, + "33": 0.76729, + "34": 0.45405, + "35": 0.90679, + "36": 0.62691, + "37": 0.80734, + "38": 0.52682, + "39": 0.61651, + "40": 0.72001, + "41": 0.97926, + "42": 0.57908, + "43": 0.59221, + "44": 0.65813, + "45": 0.7194, + "46": 0.76818, + "47": 0.82352, + "48": 1.01064, + "49": 0.36409, + "50": 0.3676, + "51": 0.41549, + "52": 0.39786, + "53": 0.64141, + "54": 0.81457, + "55": 0.47145, + "56": 0.75849, + "57": 0.47005, + "58": 0.54556, + "59": 0.65077, + "60": 0.67032, + "61": 0.36282, + "62": 1.07816, + "63": 0.36068, + "64": 0.81042, + "65": 0.43636, + "66": 0.65437, + "67": 0.61425, + "68": 0.36157, + "69": 0.60737, + "70": 1.00435, + "71": 0.35852, + "72": 0.49448, + "73": 0.69778, + "74": 0.4904, + "75": 1.02199, + "76": 0.43752, + "77": 0.42751, + "78": 1.80056, + "79": 0.65201, + "80": 0.64025, + "81": 0.64801, + "82": 0.44684, + "83": 0.8134, + "84": 0.73747, + "85": 0.35805, + "86": 0.39074, + "87": 0.95679, + "88": 0.91998, + "89": 0.64926, + "90": 0.48592, + "91": 0.55213, + "92": 0.83853, + "93": 0.44924, + "94": 0.63982, + "95": 0.64996, + "96": 0.50551, + "97": 0.47138, + "98": 0.72542, + "99": 1.35867, + "100": 0.35987 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json index 362ed2517a2..823258241f1 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90649, - "4": 10.90498, - "5": 10.91356, - "6": 10.89668, - "7": 10.89772, - "8": 10.91234, - "9": 10.89371, - "10": 10.90081, - "11": 10.89194, - "12": 10.90692, - "13": 10.8718, - "14": 10.86067, - "15": 10.87356, - "16": 10.85943, - "17": 10.84008, - "18": 10.83747, - "19": 10.84048, - "20": 10.79175, - "21": 10.75636, - "22": 10.75648, - "23": 10.75817, - "24": 10.72019, - "25": 10.72096, - "26": 10.69808, - "27": 10.68267, - "28": 10.61119, - "29": 10.59082, - "30": 10.57447, - "31": 10.56111, - "32": 10.54229, - "33": 10.51733, - "34": 10.47715, - "35": 10.48521, - "36": 10.47888, - "37": 10.42825, - "38": 10.43511, - "39": 10.40608, - "40": 10.39417, - "41": 10.37114, - "42": 10.35538, - "43": 10.32417, - "44": 10.30788, - "45": 10.31573, - "46": 10.27767, - "47": 10.26813, - "48": 10.22377, - "49": 10.21599, - "50": 10.21902, - "51": 10.22193, - "52": 10.17292, - "53": 10.17695, + "1": 10.90111, + "2": 10.8957, + "3": 10.90626, + "4": 10.90562, + "5": 10.91338, + "6": 10.89684, + "7": 10.89792, + "8": 10.91285, + "9": 10.89279, + "10": 10.90027, + "11": 10.89132, + "12": 10.90672, + "13": 10.87062, + "14": 10.86054, + "15": 10.87324, + "16": 10.85911, + "17": 10.84015, + "18": 10.83726, + "19": 10.83943, + "20": 10.79178, + "21": 10.75607, + "22": 10.75557, + "23": 10.75782, + "24": 10.72, + "25": 10.72107, + "26": 10.69764, + "27": 10.68234, + "28": 10.61163, + "29": 10.5913, + "30": 10.57438, + "31": 10.56065, + "32": 10.54263, + "33": 10.51706, + "34": 10.47723, + "35": 10.48572, + "36": 10.47878, + "37": 10.42879, + "38": 10.43527, + "39": 10.40593, + "40": 10.39405, + "41": 10.3711, + "42": 10.35553, + "43": 10.32405, + "44": 10.3079, + "45": 10.3161, + "46": 10.27786, + "47": 10.26864, + "48": 10.22395, + "49": 10.21645, + "50": 10.21926, + "51": 10.22194, + "52": 10.17303, + "53": 10.17725, "54": 10.14489, - "55": 10.11386, - "56": 10.1422, - "57": 10.12634, - "58": 10.13387, - "59": 10.07883, - "60": 10.10032, - "61": 10.0569, - "62": 10.02093, - "63": 10.0933, - "64": 10.03529, - "65": 10.00596, - "66": 10.04629, - "67": 10.01887, - "68": 9.97911, - "69": 9.99395, - "70": 9.98094, - "71": 10.00401, - "72": 9.98666, - "73": 9.96193, - "74": 9.9599, - "75": 9.91413, - "76": 9.95283, - "77": 9.95124, - "78": 9.90005, - "79": 9.90154, - "80": 9.91121, - "81": 9.94218, - "82": 9.89783, - "83": 9.85125, - "84": 9.79584, - "85": 9.77428, - "86": 9.89304, - "87": 9.90528, - "88": 9.883, - "89": 9.82604, - "90": 9.8253, - "91": 9.82873, - "92": 9.82373, - "93": 9.75213, - "94": 9.83157, - "95": 9.82108, - "96": 9.80782, - "97": 9.75363, - "98": 9.78415, - "99": 9.83389, - "100": 9.72314 + "55": 10.1139, + "56": 10.14246, + "57": 10.12615, + "58": 10.13392, + "59": 10.07842, + "60": 10.10018, + "61": 10.05693, + "62": 10.02102, + "63": 10.09347, + "64": 10.03541, + "65": 10.00572, + "66": 10.04666, + "67": 10.01884, + "68": 9.9791, + "69": 9.99387, + "70": 9.9811, + "71": 10.00411, + "72": 9.9866, + "73": 9.96195, + "74": 9.96004, + "75": 9.91415, + "76": 9.95265, + "77": 9.9512, + "78": 9.89997, + "79": 9.90165, + "80": 9.91122, + "81": 9.94213, + "82": 9.89797, + "83": 9.85138, + "84": 9.79603, + "85": 9.77409, + "86": 9.89333, + "87": 9.90529, + "88": 9.88326, + "89": 9.82622, + "90": 9.82545, + "91": 9.82891, + "92": 9.8238, + "93": 9.7521, + "94": 9.83174, + "95": 9.82115, + "96": 9.80774, + "97": 9.75361, + "98": 9.78431, + "99": 9.83399, + "100": 9.72316 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1363.0, - "4": 1284.0, - "5": 1232.0, - "6": 1243.0, - "7": 1313.0, - "8": 1225.0, - "9": 1225.0, - "10": 1193.0, - "11": 1184.0, - "12": 1158.0, - "13": 1384.0, - "14": 1251.0, - "15": 1118.0, - "16": 1300.0, - "17": 1330.0, - "18": 1189.0, - "19": 1107.0, - "20": 1077.0, - "21": 1136.0, - "22": 1123.0, - "23": 1292.0, - "24": 1187.0, - "25": 1126.0, - "26": 1256.0, - "27": 1218.0, - "28": 1134.0, - "29": 1166.0, - "30": 1154.0, - "31": 1151.0, - "32": 1176.0, - "33": 1276.0, - "34": 1108.0, - "35": 1151.0, - "36": 1134.0, - "37": 1210.0, - "38": 1192.0, - "39": 1286.0, - "40": 1221.0, - "41": 1170.0, - "42": 1177.0, - "43": 1149.0, - "44": 1224.0, - "45": 1204.0, - "46": 1271.0, - "47": 1239.0, - "48": 1307.0, - "49": 1225.0, - "50": 1288.0, - "51": 1461.0, - "52": 1330.0, - "53": 1286.0, - "54": 1422.0, - "55": 1193.0, - "56": 1364.0, - "57": 1231.0, - "58": 1050746.0, - "59": 1386.0, - "60": 1398.0, - "61": 42279.0, - "62": 49536.0, - "63": 1562.0, - "64": 42506.0, - "65": 1237.0, - "66": 1050485.0, - "67": 50645.0, - "68": 1553.0, - "69": 5459.0, - "70": 6617.0, - "71": 2356.0, - "72": 57739.0, - "73": 57912.0, - "74": 67070.0, - "75": 11773.0, - "76": 70154.0, - "77": 1701.0, - "78": 1612.0, - "79": 6649.0, - "80": 5470.0, - "81": 2645.0, - "82": 2709.0, - "83": 1410.0, - "84": 1050554.0, - "85": 1560.0, - "86": 1540.0, - "87": 2152913.0, - "88": 2420.0, - "89": 10807.0, - "90": 1060087.0, - "91": 1295.0, - "92": 2099726.0, - "93": 10800.0, - "94": 2764.0, - "95": 1050655.0, - "96": 116340.0, - "97": 1050841.0, - "98": 54863.0, - "99": 51572.0, - "100": 1050668.0 + "1": 1133.0, + "2": 1120.0, + "3": 1357.0, + "4": 1328.0, + "5": 1243.0, + "6": 1194.0, + "7": 1359.0, + "8": 1220.0, + "9": 1207.0, + "10": 1187.0, + "11": 1241.0, + "12": 1150.0, + "13": 1293.0, + "14": 1250.0, + "15": 1165.0, + "16": 1281.0, + "17": 1240.0, + "18": 1254.0, + "19": 1207.0, + "20": 1123.0, + "21": 1158.0, + "22": 1122.0, + "23": 1212.0, + "24": 1223.0, + "25": 1195.0, + "26": 1229.0, + "27": 1201.0, + "28": 1173.0, + "29": 1129.0, + "30": 1179.0, + "31": 1078.0, + "32": 1148.0, + "33": 1241.0, + "34": 1120.0, + "35": 1181.0, + "36": 1113.0, + "37": 1150.0, + "38": 1171.0, + "39": 1185.0, + "40": 1211.0, + "41": 1313.0, + "42": 1187.0, + "43": 1181.0, + "44": 1146.0, + "45": 1186.0, + "46": 1210.0, + "47": 1153.0, + "48": 1245.0, + "49": 1302.0, + "50": 1326.0, + "51": 1447.0, + "52": 1342.0, + "53": 1269.0, + "54": 1381.0, + "55": 1228.0, + "56": 1273.0, + "57": 1214.0, + "58": 50777.0, + "59": 1361.0, + "60": 1406.0, + "61": 42322.0, + "62": 49580.0, + "63": 1577.0, + "64": 43545.0, + "65": 1216.0, + "66": 1050479.0, + "67": 51707.0, + "68": 1528.0, + "69": 5467.0, + "70": 6590.0, + "71": 2418.0, + "72": 57795.0, + "73": 7777.0, + "74": 11706.0, + "75": 13832.0, + "76": 70208.0, + "77": 1681.0, + "78": 1563.0, + "79": 6703.0, + "80": 5581.0, + "81": 2703.0, + "82": 2750.0, + "83": 1486.0, + "84": 1050539.0, + "85": 1435.0, + "86": 1495.0, + "87": 2151913.0, + "88": 2377.0, + "89": 10766.0, + "90": 1060091.0, + "91": 1258.0, + "92": 2099760.0, + "93": 11822.0, + "94": 1776.0, + "95": 1050670.0, + "96": 116325.0, + "97": 53923.0, + "98": 53892.0, + "99": 5442.0, + "100": 1050643.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 788519424.0, - "2": 788509696.0, - "3": 788487680.0, - "4": 788466176.0, - "5": 788516352.0, - "6": 788500992.0, - "7": 788425216.0, - "8": 788463104.0, - "9": 788467200.0, - "10": 788497920.0, - "11": 788475904.0, - "12": 788486144.0, - "13": 788488704.0, - "14": 788493824.0, - "15": 788476928.0, - "16": 788465152.0, + "1": 788519936.0, + "2": 788510720.0, + "3": 788485120.0, + "4": 788465664.0, + "5": 788514816.0, + "6": 788501504.0, + "7": 788426240.0, + "8": 788464128.0, + "9": 788466176.0, + "10": 788499456.0, + "11": 788475392.0, + "12": 788483584.0, + "13": 788490240.0, + "14": 788493312.0, + "15": 788476416.0, + "16": 788467712.0, "17": 788476416.0, - "18": 788482560.0, - "19": 788512768.0, - "20": 788488192.0, - "21": 788554752.0, - "22": 788592128.0, - "23": 788578816.0, + "18": 788483584.0, + "19": 788512256.0, + "20": 788488704.0, + "21": 788551680.0, + "22": 788594176.0, + "23": 788579328.0, "24": 788489728.0, - "25": 788579328.0, - "26": 788623360.0, - "27": 788565504.0, - "28": 788536320.0, - "29": 788600832.0, + "25": 788578816.0, + "26": 788620288.0, + "27": 788563456.0, + "28": 788538368.0, + "29": 788597760.0, "30": 788569600.0, - "31": 788587520.0, - "32": 788580352.0, + "31": 788587008.0, + "32": 788580864.0, "33": 788574208.0, - "34": 788564992.0, - "35": 788553728.0, - "36": 788604416.0, - "37": 788551680.0, - "38": 788576768.0, - "39": 788595712.0, - "40": 788554240.0, - "41": 788546560.0, - "42": 788506112.0, - "43": 788538368.0, - "44": 788487168.0, - "45": 788457984.0, - "46": 788498944.0, - "47": 788450816.0, - "48": 788437504.0, - "49": 788416000.0, - "50": 788386816.0, - "51": 788401664.0, - "52": 788369408.0, - "53": 788322816.0, - "54": 788357120.0, - "55": 788350464.0, - "56": 788314624.0, - "57": 788388864.0, - "58": 788365312.0, + "34": 788563456.0, + "35": 788553216.0, + "36": 788605440.0, + "37": 788550144.0, + "38": 788575744.0, + "39": 788595200.0, + "40": 788552704.0, + "41": 788545536.0, + "42": 788503552.0, + "43": 788534272.0, + "44": 788486144.0, + "45": 788453888.0, + "46": 788497408.0, + "47": 788449792.0, + "48": 788434944.0, + "49": 788413952.0, + "50": 788384256.0, + "51": 788398592.0, + "52": 788370944.0, + "53": 788320256.0, + "54": 788355584.0, + "55": 788348416.0, + "56": 788313088.0, + "57": 788386816.0, + "58": 788362752.0, "59": 788348416.0, - "60": 788370944.0, - "61": 788306944.0, - "62": 788299264.0, - "63": 788331008.0, - "64": 788336640.0, - "65": 788324864.0, - "66": 788332032.0, - "67": 788337664.0, - "68": 788326912.0, - "69": 788331008.0, - "70": 788350976.0, + "60": 788368896.0, + "61": 788305920.0, + "62": 788297728.0, + "63": 788329472.0, + "64": 788335104.0, + "65": 788324352.0, + "66": 788331008.0, + "67": 788335104.0, + "68": 788327424.0, + "69": 788331520.0, + "70": 788349952.0, "71": 788344832.0, - "72": 788391424.0, - "73": 788350976.0, - "74": 788380672.0, - "75": 788320256.0, - "76": 788292096.0, - "77": 788396544.0, - "78": 788358656.0, - "79": 788318208.0, - "80": 788278272.0, - "81": 788282368.0, - "82": 788242432.0, - "83": 788274688.0, + "72": 788392960.0, + "73": 788349952.0, + "74": 788381184.0, + "75": 788319744.0, + "76": 788291584.0, + "77": 788393472.0, + "78": 788356096.0, + "79": 788316160.0, + "80": 788275712.0, + "81": 788283904.0, + "82": 788240896.0, + "83": 788277248.0, "84": 788317696.0, - "85": 788315136.0, - "86": 788338176.0, - "87": 788144640.0, - "88": 788254208.0, - "89": 788261888.0, - "90": 788246016.0, - "91": 788318208.0, - "92": 788240384.0, - "93": 788244480.0, - "94": 788385792.0, - "95": 788250112.0, - "96": 788293632.0, - "97": 788358656.0, - "98": 788327936.0, - "99": 788265472.0, - "100": 788357632.0 + "85": 788314624.0, + "86": 788337664.0, + "87": 788147200.0, + "88": 788256256.0, + "89": 788264448.0, + "90": 788247552.0, + "91": 788319232.0, + "92": 788239872.0, + "93": 788245504.0, + "94": 788387840.0, + "95": 788252672.0, + "96": 788294656.0, + "97": 788360192.0, + "98": 788326912.0, + "99": 788267008.0, + "100": 788357120.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3122096128.0, - "2": 3285577216.0, - "3": 3285577216.0, - "4": 3285577216.0, - "5": 3288475648.0, - "6": 3288475648.0, - "7": 3288475648.0, - "8": 3288475648.0, - "9": 3288475648.0, - "10": 3288475648.0, - "11": 3288475648.0, - "12": 3288475648.0, - "13": 3288475648.0, - "14": 3288475648.0, - "15": 3288475648.0, - "16": 3288475648.0, - "17": 3288475648.0, - "18": 3288475648.0, - "19": 3288475648.0, - "20": 3288475648.0, - "21": 3315479040.0, - "22": 3342354432.0, - "23": 3342354432.0, - "24": 3342354432.0, - "25": 3342354432.0, - "26": 3359620608.0, - "27": 3359620608.0, - "28": 3359620608.0, - "29": 3359620608.0, - "30": 3359620608.0, - "31": 3359620608.0, - "32": 3359620608.0, - "33": 3359620608.0, - "34": 3359620608.0, - "35": 3359620608.0, - "36": 3359620608.0, - "37": 3359620608.0, - "38": 3359620608.0, - "39": 3359620608.0, - "40": 3359620608.0, - "41": 3359620608.0, - "42": 3359620608.0, - "43": 3359620608.0, - "44": 3359620608.0, - "45": 3359620608.0, - "46": 3359620608.0, - "47": 3359620608.0, - "48": 3359620608.0, - "49": 3359620608.0, - "50": 3359620608.0, - "51": 3359620608.0, - "52": 3359620608.0, - "53": 3359620608.0, - "54": 3359620608.0, - "55": 3359620608.0, - "56": 3359620608.0, - "57": 3359620608.0, - "58": 3359620608.0, - "59": 3359620608.0, - "60": 3359620608.0, - "61": 3359620608.0, - "62": 3359620608.0, - "63": 3359620608.0, - "64": 3359620608.0, - "65": 3359620608.0, - "66": 3359620608.0, - "67": 3359620608.0, - "68": 3359620608.0, - "69": 3359620608.0, - "70": 3359620608.0, - "71": 3359620608.0, - "72": 3359620608.0, - "73": 3359620608.0, - "74": 3359620608.0, - "75": 3359620608.0, - "76": 3359620608.0, - "77": 3359620608.0, - "78": 3359620608.0, - "79": 3359620608.0, - "80": 3359620608.0, - "81": 3359620608.0, - "82": 3359620608.0, - "83": 3359620608.0, - "84": 3359620608.0, - "85": 3359620608.0, - "86": 3359620608.0, - "87": 3359620608.0, - "88": 3359620608.0, - "89": 3359620608.0, - "90": 3359620608.0, - "91": 3359620608.0, - "92": 3359620608.0, - "93": 3359620608.0, - "94": 3359620608.0, - "95": 3359620608.0, - "96": 3359620608.0, - "97": 3359620608.0, - "98": 3359620608.0, - "99": 3359620608.0, - "100": 3359620608.0 + "1": 3121236480.0, + "2": 3284394496.0, + "3": 3284394496.0, + "4": 3284394496.0, + "5": 3289592320.0, + "6": 3289592320.0, + "7": 3289592320.0, + "8": 3289592320.0, + "9": 3289592320.0, + "10": 3289592320.0, + "11": 3289592320.0, + "12": 3289592320.0, + "13": 3289592320.0, + "14": 3289592320.0, + "15": 3289592320.0, + "16": 3289592320.0, + "17": 3289592320.0, + "18": 3289592320.0, + "19": 3289592320.0, + "20": 3289592320.0, + "21": 3315035648.0, + "22": 3342148608.0, + "23": 3342148608.0, + "24": 3342148608.0, + "25": 3342148608.0, + "26": 3359425024.0, + "27": 3359425024.0, + "28": 3359425024.0, + "29": 3359425024.0, + "30": 3359425024.0, + "31": 3359425024.0, + "32": 3359425024.0, + "33": 3359425024.0, + "34": 3359425024.0, + "35": 3359425024.0, + "36": 3359425024.0, + "37": 3359425024.0, + "38": 3359425024.0, + "39": 3359425024.0, + "40": 3359425024.0, + "41": 3359425024.0, + "42": 3359425024.0, + "43": 3359425024.0, + "44": 3359425024.0, + "45": 3359425024.0, + "46": 3359425024.0, + "47": 3359425024.0, + "48": 3359425024.0, + "49": 3359425024.0, + "50": 3359425024.0, + "51": 3359425024.0, + "52": 3359425024.0, + "53": 3359425024.0, + "54": 3359425024.0, + "55": 3359425024.0, + "56": 3359425024.0, + "57": 3359425024.0, + "58": 3359425024.0, + "59": 3359425024.0, + "60": 3359425024.0, + "61": 3359425024.0, + "62": 3359425024.0, + "63": 3359425024.0, + "64": 3359425024.0, + "65": 3359425024.0, + "66": 3359425024.0, + "67": 3359425024.0, + "68": 3359425024.0, + "69": 3359425024.0, + "70": 3359425024.0, + "71": 3359425024.0, + "72": 3359425024.0, + "73": 3359425024.0, + "74": 3359425024.0, + "75": 3359425024.0, + "76": 3359425024.0, + "77": 3359425024.0, + "78": 3359425024.0, + "79": 3359425024.0, + "80": 3359425024.0, + "81": 3359425024.0, + "82": 3359425024.0, + "83": 3359425024.0, + "84": 3359425024.0, + "85": 3359425024.0, + "86": 3359425024.0, + "87": 3359425024.0, + "88": 3359425024.0, + "89": 3359425024.0, + "90": 3359425024.0, + "91": 3359425024.0, + "92": 3359425024.0, + "93": 3359425024.0, + "94": 3359425024.0, + "95": 3359425024.0, + "96": 3359425024.0, + "97": 3359425024.0, + "98": 3359425024.0, + "99": 3359425024.0, + "100": 3359425024.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.18868, - "3": 0.18976, - "4": 0.18601, - "5": 0.178, - "6": 0.17013, - "7": 0.17369, - "8": 0.18316, - "9": 0.16018, - "10": 0.1641, - "11": 0.17079, - "12": 0.16346, - "13": 0.1628, - "14": 0.16331, - "15": 0.15023, - "16": 0.15341, - "17": 0.15639, - "18": 0.15465, - "19": 0.14892, - "20": 0.15916, - "21": 0.15901, - "22": 0.15961, - "23": 0.15706, - "24": 0.16439, - "25": 0.17181, - "26": 0.15735, - "27": 0.17408, - "28": 0.17844, - "29": 0.17656, - "30": 0.1684, - "31": 0.15774, - "32": 0.18388, - "33": 0.17033, - "34": 0.17365, - "35": 0.16585, - "36": 0.17175, - "37": 0.16836, - "38": 0.16953, - "39": 0.17684, - "40": 0.1544, - "41": 0.16069, - "42": 0.16297, - "43": 0.17956, - "44": 0.16333, - "45": 0.16491, - "46": 0.15417, - "47": 0.15747, - "48": 0.15582, - "49": 0.15871, - "50": 0.15837, - "51": 0.20584, - "52": 0.19572, - "53": 0.15813, - "54": 0.15301, - "55": 0.15257, - "56": 0.14771, - "57": 0.16512, - "58": 0.15268, - "59": 0.15716, - "60": 0.16019, - "61": 0.15621, - "62": 0.16853, - "63": 0.15277, - "64": 0.14379, - "65": 0.14892, - "66": 0.15734, - "67": 0.15021, - "68": 0.15207, - "69": 0.15622, - "70": 0.14784, - "71": 0.14597, - "72": 0.15762, - "73": 0.15952, - "74": 0.15146, - "75": 0.15538, - "76": 0.14898, - "77": 0.15001, - "78": 0.1499, - "79": 0.15411, - "80": 0.1511, - "81": 0.14847, - "82": 0.14689, - "83": 0.14637, - "84": 0.14672, - "85": 0.14407, - "86": 0.15642, - "87": 0.14024, - "88": 0.15645, - "89": 0.14898, - "90": 0.14543, - "91": 0.14593, - "92": 0.14214, - "93": 0.14251, - "94": 0.14648, - "95": 0.15641, - "96": 0.14005, - "97": 0.14035, - "98": 0.14839, - "99": 0.12987, - "100": 0.13898 + "2": 4.52323, + "3": 0.19663, + "4": 0.17543, + "5": 0.19069, + "6": 0.17051, + "7": 0.17616, + "8": 0.18284, + "9": 0.17765, + "10": 0.17291, + "11": 0.18329, + "12": 0.15973, + "13": 0.17233, + "14": 0.16664, + "15": 0.16783, + "16": 0.16783, + "17": 0.17292, + "18": 0.15603, + "19": 0.15638, + "20": 0.16092, + "21": 0.17656, + "22": 0.1677, + "23": 0.15908, + "24": 0.16291, + "25": 0.184, + "26": 0.15922, + "27": 0.17381, + "28": 0.18227, + "29": 0.1705, + "30": 0.17401, + "31": 0.1729, + "32": 0.18045, + "33": 0.17736, + "34": 0.18122, + "35": 0.15835, + "36": 0.17971, + "37": 0.17994, + "38": 0.18053, + "39": 0.18529, + "40": 0.16901, + "41": 0.16177, + "42": 0.17084, + "43": 0.16807, + "44": 0.16785, + "45": 0.16278, + "46": 0.17404, + "47": 0.16869, + "48": 0.15239, + "49": 0.15444, + "50": 0.16228, + "51": 1.71136, + "52": 0.19743, + "53": 0.22424, + "54": 0.15535, + "55": 0.15493, + "56": 0.14309, + "57": 0.16908, + "58": 0.21236, + "59": 0.15781, + "60": 0.21784, + "61": 0.14226, + "62": 0.17439, + "63": 0.22118, + "64": 0.15047, + "65": 0.16162, + "66": 0.17192, + "67": 0.15749, + "68": 0.14332, + "69": 0.16492, + "70": 0.15042, + "71": 0.17996, + "72": 0.15294, + "73": 0.21192, + "74": 0.15977, + "75": 0.15858, + "76": 0.15006, + "77": 0.16115, + "78": 0.15397, + "79": 0.15516, + "80": 0.14733, + "81": 0.15321, + "82": 0.15755, + "83": 0.15836, + "84": 0.15737, + "85": 0.1536, + "86": 0.15088, + "87": 0.147, + "88": 0.15501, + "89": 0.15595, + "90": 0.14272, + "91": 0.15399, + "92": 0.1428, + "93": 0.15036, + "94": 0.14851, + "95": 0.14892, + "96": 0.14683, + "97": 0.15235, + "98": 0.14893, + "99": 0.14814, + "100": 0.14019 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index c31c834f04c..642e2c5b5bd 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.92513, - "4": 10.91739, - "5": 10.9269, - "6": 10.92031, - "7": 10.91598, - "8": 10.91649, - "9": 10.93519, - "10": 10.9206, - "11": 10.90717, - "12": 10.91306, - "13": 10.92203, - "14": 10.90168, - "15": 10.89219, - "16": 10.87582, - "17": 10.88267, - "18": 10.85858, - "19": 10.87882, - "20": 10.80364, - "21": 10.81054, - "22": 10.78654, - "23": 10.77822, - "24": 10.74148, - "25": 10.73996, - "26": 10.73202, - "27": 10.71123, - "28": 10.65525, - "29": 10.62359, - "30": 10.59735, - "31": 10.58089, - "32": 10.57395, - "33": 10.54555, - "34": 10.50091, - "35": 10.51208, - "36": 10.48236, - "37": 10.45341, - "38": 10.46938, - "39": 10.42943, - "40": 10.41557, - "41": 10.3926, - "42": 10.38242, - "43": 10.34965, - "44": 10.33277, - "45": 10.33406, - "46": 10.30638, - "47": 10.28401, - "48": 10.2469, - "49": 10.2361, - "50": 10.24807, - "51": 10.24514, - "52": 10.19574, - "53": 10.19181, - "54": 10.16588, - "55": 10.12867, - "56": 10.16541, - "57": 10.13627, - "58": 10.15663, - "59": 10.10169, - "60": 10.1085, - "61": 10.07239, - "62": 10.03691, - "63": 10.10933, - "64": 10.05782, - "65": 10.02658, - "66": 10.05862, - "67": 10.02843, - "68": 9.98898, - "69": 10.00162, - "70": 9.99443, - "71": 10.02546, - "72": 9.98142, - "73": 9.9673, - "74": 9.95642, - "75": 9.92751, - "76": 9.97435, - "77": 9.96221, - "78": 9.9049, - "79": 9.91517, - "80": 9.92522, - "81": 9.94341, - "82": 9.90375, - "83": 9.85642, - "84": 9.79325, - "85": 9.77579, - "86": 9.89058, - "87": 9.9164, - "88": 9.89322, - "89": 9.82868, - "90": 9.82075, - "91": 9.83171, - "92": 9.82545, - "93": 9.75863, - "94": 9.83386, - "95": 9.83104, - "96": 9.81382, - "97": 9.75302, - "98": 9.78874, - "99": 9.8339, - "100": 9.72866 + "1": 10.93064, + "2": 10.92264, + "3": 10.92441, + "4": 10.91686, + "5": 10.92701, + "6": 10.92079, + "7": 10.91574, + "8": 10.91647, + "9": 10.93458, + "10": 10.92054, + "11": 10.90683, + "12": 10.91327, + "13": 10.9214, + "14": 10.90157, + "15": 10.89311, + "16": 10.87614, + "17": 10.88256, + "18": 10.85905, + "19": 10.87765, + "20": 10.80275, + "21": 10.81075, + "22": 10.78656, + "23": 10.77818, + "24": 10.74186, + "25": 10.73977, + "26": 10.73189, + "27": 10.71152, + "28": 10.65527, + "29": 10.62335, + "30": 10.59679, + "31": 10.58051, + "32": 10.57392, + "33": 10.5458, + "34": 10.5016, + "35": 10.51196, + "36": 10.48208, + "37": 10.45398, + "38": 10.46925, + "39": 10.42854, + "40": 10.41547, + "41": 10.39223, + "42": 10.38216, + "43": 10.34944, + "44": 10.33256, + "45": 10.33374, + "46": 10.30632, + "47": 10.2841, + "48": 10.24662, + "49": 10.23591, + "50": 10.24832, + "51": 10.24533, + "52": 10.19596, + "53": 10.19195, + "54": 10.16609, + "55": 10.12861, + "56": 10.16556, + "57": 10.13618, + "58": 10.15698, + "59": 10.10159, + "60": 10.10852, + "61": 10.07214, + "62": 10.03671, + "63": 10.10923, + "64": 10.05778, + "65": 10.02626, + "66": 10.05854, + "67": 10.02842, + "68": 9.98887, + "69": 10.00132, + "70": 9.99481, + "71": 10.02538, + "72": 9.98121, + "73": 9.96714, + "74": 9.95643, + "75": 9.92737, + "76": 9.97455, + "77": 9.96204, + "78": 9.90473, + "79": 9.91522, + "80": 9.92511, + "81": 9.94349, + "82": 9.9036, + "83": 9.85656, + "84": 9.79289, + "85": 9.77545, + "86": 9.89051, + "87": 9.91663, + "88": 9.89297, + "89": 9.82866, + "90": 9.82093, + "91": 9.83177, + "92": 9.82526, + "93": 9.75885, + "94": 9.8339, + "95": 9.83071, + "96": 9.8138, + "97": 9.75301, + "98": 9.7888, + "99": 9.83387, + "100": 9.72864 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 48952.0, - "4": 48834.0, - "5": 50218.0, - "6": 50021.0, - "7": 51375.0, - "8": 49745.0, - "9": 51326.0, - "10": 52144.0, - "11": 49913.0, - "12": 48458.0, - "13": 51932.0, - "14": 50123.0, - "15": 47949.0, - "16": 49843.0, - "17": 49882.0, - "18": 52196.0, - "19": 51092.0, - "20": 52823.0, - "21": 49874.0, - "22": 49685.0, - "23": 53705.0, - "24": 52981.0, - "25": 49745.0, - "26": 53405.0, - "27": 53055.0, - "28": 49700.0, - "29": 52803.0, - "30": 54243.0, - "31": 56717.0, - "32": 53359.0, - "33": 56818.0, - "34": 53753.0, - "35": 55935.0, - "36": 57567.0, - "37": 54585.0, - "38": 55711.0, - "39": 58747.0, - "40": 58084.0, - "41": 55456.0, - "42": 57851.0, - "43": 61712.0, - "44": 58241.0, - "45": 67706.0, - "46": 57622.0, - "47": 105431.0, - "48": 64577.0, - "49": 71094.0, - "50": 67123.0, - "51": 63642.0, - "52": 71395.0, - "53": 110439.0, - "54": 72921.0, - "55": 61905.0, - "56": 1115059.0, - "57": 105088.0, - "58": 1127666.0, - "59": 80324.0, - "60": 123288.0, - "61": 1114978.0, - "62": 2170919.0, - "63": 70744.0, - "64": 91071.0, - "65": 173088.0, - "66": 1120234.0, - "67": 139759.0, - "68": 1123171.0, - "69": 1163608.0, - "70": 193228.0, - "71": 1118079.0, - "72": 2177823.0, - "73": 1182410.0, - "74": 2166520.0, - "75": 2166182.0, - "76": 1129053.0, - "77": 2168343.0, - "78": 1177493.0, - "79": 1175756.0, - "80": 1176743.0, - "81": 2170930.0, - "82": 2170395.0, - "83": 2170477.0, - "84": 1122782.0, - "85": 144101.0, - "86": 1124182.0, - "87": 1171244.0, - "88": 1168022.0, - "89": 1184028.0, - "90": 1183941.0, - "91": 1164459.0, - "92": 1131381.0, - "93": 1120131.0, - "94": 2175177.0, - "95": 2225361.0, - "96": 2172323.0, - "97": 2176261.0, - "98": 2175520.0, - "99": 1172004.0, - "100": 1197756.0 + "1": 49180.0, + "2": 50476.0, + "3": 49242.0, + "4": 49191.0, + "5": 50171.0, + "6": 50251.0, + "7": 51448.0, + "8": 49384.0, + "9": 50604.0, + "10": 52259.0, + "11": 50259.0, + "12": 48576.0, + "13": 51717.0, + "14": 50076.0, + "15": 48052.0, + "16": 49556.0, + "17": 49957.0, + "18": 51611.0, + "19": 50958.0, + "20": 52961.0, + "21": 50270.0, + "22": 50221.0, + "23": 54450.0, + "24": 52688.0, + "25": 49964.0, + "26": 53412.0, + "27": 52953.0, + "28": 50006.0, + "29": 52765.0, + "30": 54045.0, + "31": 56577.0, + "32": 53583.0, + "33": 56926.0, + "34": 53619.0, + "35": 55954.0, + "36": 57810.0, + "37": 54727.0, + "38": 55194.0, + "39": 59013.0, + "40": 58378.0, + "41": 55667.0, + "42": 58064.0, + "43": 61631.0, + "44": 57353.0, + "45": 68423.0, + "46": 58277.0, + "47": 105279.0, + "48": 68157.0, + "49": 70699.0, + "50": 67311.0, + "51": 63481.0, + "52": 71358.0, + "53": 110422.0, + "54": 72729.0, + "55": 110560.0, + "56": 1115272.0, + "57": 64834.0, + "58": 125706.0, + "59": 80693.0, + "60": 1116131.0, + "61": 1115149.0, + "62": 1126726.0, + "63": 71035.0, + "64": 146028.0, + "65": 173156.0, + "66": 1120354.0, + "67": 190880.0, + "68": 1122978.0, + "69": 1164011.0, + "70": 193177.0, + "71": 1118204.0, + "72": 2177337.0, + "73": 1181882.0, + "74": 2166710.0, + "75": 2165814.0, + "76": 1128584.0, + "77": 2168839.0, + "78": 1177721.0, + "79": 1176109.0, + "80": 1176668.0, + "81": 2170594.0, + "82": 1174064.0, + "83": 2170688.0, + "84": 1123127.0, + "85": 197488.0, + "86": 1124226.0, + "87": 1171138.0, + "88": 2169625.0, + "89": 1184133.0, + "90": 1183124.0, + "91": 1218390.0, + "92": 1131755.0, + "93": 1127776.0, + "94": 2175430.0, + "95": 2173930.0, + "96": 2169269.0, + "97": 2176014.0, + "98": 2175927.0, + "99": 1171444.0, + "100": 1197824.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1059404800.0, - "2": 1059426304.0, - "3": 1059409408.0, + "1": 1059406336.0, + "2": 1059425280.0, + "3": 1059410432.0, "4": 1059407360.0, - "5": 1059418624.0, - "6": 1059404288.0, - "7": 1059398656.0, + "5": 1059418112.0, + "6": 1059404800.0, + "7": 1059400704.0, "8": 1059425280.0, "9": 1059375104.0, "10": 1059374080.0, "11": 1059355648.0, - "12": 1059381248.0, - "13": 1059381248.0, - "14": 1059368448.0, + "12": 1059383808.0, + "13": 1059383296.0, + "14": 1059369984.0, "15": 1059378688.0, - "16": 1059364864.0, - "17": 1059371520.0, - "18": 1059343872.0, - "19": 1059335680.0, - "20": 1059316224.0, - "21": 1059302912.0, - "22": 1059234304.0, + "16": 1059364352.0, + "17": 1059372544.0, + "18": 1059341824.0, + "19": 1059337728.0, + "20": 1059316736.0, + "21": 1059302400.0, + "22": 1059233280.0, "23": 1059273216.0, - "24": 1059272704.0, - "25": 1059232768.0, - "26": 1059252736.0, - "27": 1059254272.0, - "28": 1059177984.0, - "29": 1059168256.0, - "30": 1059190272.0, - "31": 1059146752.0, - "32": 1059154944.0, - "33": 1059131392.0, - "34": 1059156992.0, - "35": 1059128320.0, - "36": 1059153920.0, - "37": 1059128320.0, - "38": 1059129344.0, - "39": 1059120128.0, + "24": 1059275264.0, + "25": 1059232256.0, + "26": 1059250688.0, + "27": 1059253760.0, + "28": 1059179520.0, + "29": 1059169792.0, + "30": 1059188736.0, + "31": 1059148288.0, + "32": 1059156480.0, + "33": 1059130368.0, + "34": 1059158016.0, + "35": 1059126272.0, + "36": 1059152896.0, + "37": 1059129344.0, + "38": 1059130368.0, + "39": 1059122176.0, "40": 1059153408.0, - "41": 1059121152.0, - "42": 1059118592.0, - "43": 1059116032.0, - "44": 1059122176.0, - "45": 1059144192.0, - "46": 1059119104.0, - "47": 1059133440.0, - "48": 1059131392.0, - "49": 1059138560.0, - "50": 1059123712.0, - "51": 1059121664.0, - "52": 1059122688.0, - "53": 1059100160.0, - "54": 1059146752.0, - "55": 1059116544.0, - "56": 1059119104.0, - "57": 1059092992.0, - "58": 1059114496.0, - "59": 1059089920.0, - "60": 1059072000.0, - "61": 1059088384.0, - "62": 1059093504.0, - "63": 1059085312.0, - "64": 1059065856.0, - "65": 1059073024.0, - "66": 1059051520.0, - "67": 1059049984.0, - "68": 1059025408.0, - "69": 1059041280.0, + "41": 1059123712.0, + "42": 1059117056.0, + "43": 1059117056.0, + "44": 1059122688.0, + "45": 1059145728.0, + "46": 1059118592.0, + "47": 1059131904.0, + "48": 1059131904.0, + "49": 1059140096.0, + "50": 1059124736.0, + "51": 1059123200.0, + "52": 1059125248.0, + "53": 1059101696.0, + "54": 1059148800.0, + "55": 1059118592.0, + "56": 1059121152.0, + "57": 1059094528.0, + "58": 1059115008.0, + "59": 1059092480.0, + "60": 1059071488.0, + "61": 1059089408.0, + "62": 1059096064.0, + "63": 1059086336.0, + "64": 1059067392.0, + "65": 1059074048.0, + "66": 1059052544.0, + "67": 1059052544.0, + "68": 1059028480.0, + "69": 1059041792.0, "70": 1059020288.0, "71": 1059035136.0, - "72": 1059006464.0, - "73": 1059009536.0, - "74": 1059009024.0, - "75": 1059028480.0, - "76": 1059019264.0, - "77": 1059038208.0, - "78": 1059022848.0, + "72": 1059005952.0, + "73": 1059011072.0, + "74": 1059010560.0, + "75": 1059030016.0, + "76": 1059020288.0, + "77": 1059036160.0, + "78": 1059024896.0, "79": 1059030016.0, "80": 1059014656.0, - "81": 1059033600.0, - "82": 1059005952.0, - "83": 1059012096.0, - "84": 1059024384.0, - "85": 1059003904.0, - "86": 1059016192.0, - "87": 1059000320.0, - "88": 1058989056.0, + "81": 1059033088.0, + "82": 1059005440.0, + "83": 1059007488.0, + "84": 1059025920.0, + "85": 1059004416.0, + "86": 1059015680.0, + "87": 1058998272.0, + "88": 1058990080.0, "89": 1059011584.0, - "90": 1059018752.0, - "91": 1058996736.0, - "92": 1059015680.0, - "93": 1058995712.0, - "94": 1058995200.0, - "95": 1059003392.0, - "96": 1059005440.0, - "97": 1059012096.0, - "98": 1059019264.0, - "99": 1058999808.0, - "100": 1058989568.0 + "90": 1059017216.0, + "91": 1058998272.0, + "92": 1059014656.0, + "93": 1058994176.0, + "94": 1058994176.0, + "95": 1059005952.0, + "96": 1059005952.0, + "97": 1059014144.0, + "98": 1059020288.0, + "99": 1059002368.0, + "100": 1058991104.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3362050048.0, - "2": 3674120704.0, - "3": 3674120704.0, - "4": 3674120704.0, - "5": 3674120704.0, - "6": 3674120704.0, - "7": 3674120704.0, - "8": 3674120704.0, - "9": 3674120704.0, - "10": 3674120704.0, - "11": 3674120704.0, - "12": 3674120704.0, - "13": 3674120704.0, - "14": 3674120704.0, - "15": 3674120704.0, - "16": 3674120704.0, - "17": 3674120704.0, - "18": 3674120704.0, - "19": 3674120704.0, - "20": 3674120704.0, - "21": 3674120704.0, - "22": 3674120704.0, - "23": 3674120704.0, - "24": 3674120704.0, - "25": 3674120704.0, - "26": 3674120704.0, - "27": 3674120704.0, - "28": 3674120704.0, - "29": 3674120704.0, - "30": 3674120704.0, - "31": 3674120704.0, - "32": 3674120704.0, - "33": 3674120704.0, - "34": 3674120704.0, - "35": 3674120704.0, - "36": 3674120704.0, - "37": 3674120704.0, - "38": 3674120704.0, - "39": 3674120704.0, - "40": 3674120704.0, - "41": 3674120704.0, - "42": 3674120704.0, - "43": 3674120704.0, - "44": 3674120704.0, - "45": 3674120704.0, - "46": 3674120704.0, - "47": 3674120704.0, - "48": 3674120704.0, - "49": 3674120704.0, - "50": 3674120704.0, - "51": 3674120704.0, - "52": 3674120704.0, - "53": 3674120704.0, - "54": 3674120704.0, - "55": 3674120704.0, - "56": 3674120704.0, - "57": 3674120704.0, - "58": 3674120704.0, - "59": 3674120704.0, - "60": 3674120704.0, - "61": 3674120704.0, - "62": 3674120704.0, - "63": 3674120704.0, - "64": 3674120704.0, - "65": 3674120704.0, - "66": 3674120704.0, - "67": 3674120704.0, - "68": 3674120704.0, - "69": 3674120704.0, - "70": 3674120704.0, - "71": 3674120704.0, - "72": 3674120704.0, - "73": 3674120704.0, - "74": 3674120704.0, - "75": 3674120704.0, - "76": 3674120704.0, - "77": 3674120704.0, - "78": 3674120704.0, - "79": 3674120704.0, - "80": 3674120704.0, - "81": 3674120704.0, - "82": 3674120704.0, - "83": 3674120704.0, - "84": 3674120704.0, - "85": 3674120704.0, - "86": 3674120704.0, - "87": 3674120704.0, - "88": 3674120704.0, - "89": 3674120704.0, - "90": 3674120704.0, - "91": 3674120704.0, - "92": 3674120704.0, - "93": 3674120704.0, - "94": 3674120704.0, - "95": 3674120704.0, - "96": 3674120704.0, - "97": 3674120704.0, - "98": 3674120704.0, - "99": 3674120704.0, - "100": 3674120704.0 + "1": 3362376192.0, + "2": 3675800064.0, + "3": 3675800064.0, + "4": 3675800064.0, + "5": 3675800064.0, + "6": 3675800064.0, + "7": 3675800064.0, + "8": 3675800064.0, + "9": 3675800064.0, + "10": 3675800064.0, + "11": 3675800064.0, + "12": 3675800064.0, + "13": 3675800064.0, + "14": 3675800064.0, + "15": 3675800064.0, + "16": 3675800064.0, + "17": 3675800064.0, + "18": 3675800064.0, + "19": 3675800064.0, + "20": 3675800064.0, + "21": 3675800064.0, + "22": 3675800064.0, + "23": 3675800064.0, + "24": 3675800064.0, + "25": 3675800064.0, + "26": 3675800064.0, + "27": 3675800064.0, + "28": 3675800064.0, + "29": 3675800064.0, + "30": 3675800064.0, + "31": 3675800064.0, + "32": 3675800064.0, + "33": 3675800064.0, + "34": 3675800064.0, + "35": 3675800064.0, + "36": 3675800064.0, + "37": 3675800064.0, + "38": 3675800064.0, + "39": 3675800064.0, + "40": 3675800064.0, + "41": 3675800064.0, + "42": 3675800064.0, + "43": 3675800064.0, + "44": 3675800064.0, + "45": 3675800064.0, + "46": 3675800064.0, + "47": 3675800064.0, + "48": 3675800064.0, + "49": 3675800064.0, + "50": 3675800064.0, + "51": 3675800064.0, + "52": 3675800064.0, + "53": 3675800064.0, + "54": 3675800064.0, + "55": 3675800064.0, + "56": 3675800064.0, + "57": 3675800064.0, + "58": 3675800064.0, + "59": 3675800064.0, + "60": 3675800064.0, + "61": 3675800064.0, + "62": 3675800064.0, + "63": 3675800064.0, + "64": 3675800064.0, + "65": 3675800064.0, + "66": 3675800064.0, + "67": 3675800064.0, + "68": 3675800064.0, + "69": 3675800064.0, + "70": 3675800064.0, + "71": 3675800064.0, + "72": 3675800064.0, + "73": 3675800064.0, + "74": 3675800064.0, + "75": 3675800064.0, + "76": 3675800064.0, + "77": 3675800064.0, + "78": 3675800064.0, + "79": 3675800064.0, + "80": 3675800064.0, + "81": 3675800064.0, + "82": 3675800064.0, + "83": 3675800064.0, + "84": 3675800064.0, + "85": 3675800064.0, + "86": 3675800064.0, + "87": 3675800064.0, + "88": 3675800064.0, + "89": 3675800064.0, + "90": 3675800064.0, + "91": 3675800064.0, + "92": 3675800064.0, + "93": 3675800064.0, + "94": 3675800064.0, + "95": 3675800064.0, + "96": 3675800064.0, + "97": 3675800064.0, + "98": 3675800064.0, + "99": 3675800064.0, + "100": 3675800064.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.35509, - "3": 0.3679, - "4": 0.4669, - "5": 0.45112, - "6": 0.56785, - "7": 0.92038, - "8": 1.0716, - "9": 0.68718, - "10": 0.37326, - "11": 1.22305, - "12": 0.35209, - "13": 0.70298, - "14": 0.77455, - "15": 0.38536, - "16": 0.85355, - "17": 0.60318, - "18": 1.00323, - "19": 0.32263, - "20": 0.53113, - "21": 0.67641, - "22": 0.63731, - "23": 0.74937, - "24": 0.32413, - "25": 0.53498, - "26": 1.10121, - "27": 0.33237, - "28": 0.54695, - "29": 0.86874, - "30": 0.32495, - "31": 0.31728, - "32": 1.03664, - "33": 0.57349, - "34": 0.32496, - "35": 0.72146, - "36": 0.60269, - "37": 0.59376, - "38": 0.43416, - "39": 0.64967, - "40": 0.79935, - "41": 0.56236, - "42": 0.69708, - "43": 0.51618, - "44": 0.9736, - "45": 0.51171, - "46": 0.77559, - "47": 0.62083, - "48": 0.9472, - "49": 0.5105, - "50": 0.42938, - "51": 0.42508, - "52": 0.37932, - "53": 0.43119, - "54": 0.8941, - "55": 0.51209, - "56": 0.83069, - "57": 0.43623, - "58": 0.35817, - "59": 0.81733, - "60": 0.63479, - "61": 0.31775, - "62": 0.72748, - "63": 0.58134, - "64": 0.91265, - "65": 1.01681, - "66": 0.99293, - "67": 0.73101, - "68": 0.56925, - "69": 1.43381, - "70": 1.08257, - "71": 0.31943, - "72": 0.61016, - "73": 1.05691, - "74": 0.51477, - "75": 1.5774, - "76": 0.44788, - "77": 0.54714, - "78": 1.57928, - "79": 0.49848, - "80": 0.63411, - "81": 0.67121, - "82": 0.72404, - "83": 0.56718, - "84": 0.78362, - "85": 0.31149, - "86": 0.45683, - "87": 1.58911, - "88": 0.53302, - "89": 0.53568, - "90": 0.54116, - "91": 0.57453, - "92": 1.05594, - "93": 0.32571, - "94": 0.68233, - "95": 0.68219, - "96": 0.65202, - "97": 0.35559, - "98": 0.4391, - "99": 1.42006, - "100": 0.33472 + "2": 5.8544, + "3": 0.33479, + "4": 0.84529, + "5": 0.41803, + "6": 0.62979, + "7": 0.66614, + "8": 0.98812, + "9": 0.62673, + "10": 0.65831, + "11": 1.45441, + "12": 0.29222, + "13": 0.69402, + "14": 0.7904, + "15": 0.41319, + "16": 0.9239, + "17": 0.57017, + "18": 0.88246, + "19": 0.29721, + "20": 0.53598, + "21": 0.75701, + "22": 0.65458, + "23": 0.6379, + "24": 0.31479, + "25": 0.63535, + "26": 1.00085, + "27": 0.29889, + "28": 0.5651, + "29": 0.80285, + "30": 0.3138, + "31": 0.30151, + "32": 0.90009, + "33": 0.56187, + "34": 0.30414, + "35": 0.73858, + "36": 0.53432, + "37": 0.5973, + "38": 0.68028, + "39": 0.84754, + "40": 0.87166, + "41": 0.61296, + "42": 0.71257, + "43": 0.61409, + "44": 0.68042, + "45": 0.68354, + "46": 0.48259, + "47": 0.59274, + "48": 0.84108, + "49": 1.01874, + "50": 0.68615, + "51": 0.42487, + "52": 0.40695, + "53": 0.36344, + "54": 0.92318, + "55": 0.52718, + "56": 0.89162, + "57": 0.4187, + "58": 0.40775, + "59": 0.79206, + "60": 0.70191, + "61": 0.42243, + "62": 0.7524, + "63": 0.56173, + "64": 0.79395, + "65": 0.47997, + "66": 0.77395, + "67": 0.61785, + "68": 0.29752, + "69": 0.75958, + "70": 1.0074, + "71": 0.30037, + "72": 0.57928, + "73": 0.6101, + "74": 0.50353, + "75": 0.98506, + "76": 0.44574, + "77": 0.40335, + "78": 1.38062, + "79": 0.326, + "80": 0.68237, + "81": 0.96378, + "82": 0.38375, + "83": 0.92744, + "84": 0.85441, + "85": 0.36274, + "86": 1.36667, + "87": 0.50849, + "88": 0.85468, + "89": 0.7165, + "90": 0.44321, + "91": 1.01346, + "92": 1.40037, + "93": 0.47205, + "94": 0.37823, + "95": 1.47369, + "96": 0.48369, + "97": 0.46675, + "98": 0.70251, + "99": 1.57993, + "100": 0.34129 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json index 709dea2488b..6262d8980dc 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.90136, - "2": 10.89626, - "3": 10.90652, - "4": 10.90542, - "5": 10.91347, - "6": 10.89722, - "7": 10.89815, - "8": 10.91325, - "9": 10.89596, - "10": 10.90411, - "11": 10.9007, - "12": 10.91797, - "13": 10.88426, - "14": 10.87521, - "15": 10.90976, - "16": 10.90312, - "17": 10.88662, - "18": 10.89386, - "19": 10.89448, - "20": 10.89126, - "21": 10.88678, - "22": 10.90299, - "23": 10.91234, - "24": 10.88568, - "25": 10.90086, - "26": 10.88913, - "27": 10.89694, - "28": 10.88118, - "29": 10.88993, - "30": 10.9105, - "31": 10.89106, - "32": 10.88858, - "33": 10.8978, - "34": 10.87329, - "35": 10.89659, - "36": 10.90752, - "37": 10.871, - "38": 10.87897, - "39": 10.88601, - "40": 10.89213, - "41": 10.88357, - "42": 10.89522, - "43": 10.88028, - "44": 10.88496, - "45": 10.88315, - "46": 10.88638, - "47": 10.88589, - "48": 10.86391, - "49": 10.87629, - "50": 10.8831, - "51": 10.89327, - "52": 10.8706, - "53": 10.8583, - "54": 10.87089, - "55": 10.8681, - "56": 10.87353, - "57": 10.84429, - "58": 10.85622, - "59": 10.84583, - "60": 10.84335, - "61": 10.86109, - "62": 10.85723, - "63": 10.86005, - "64": 10.83881, - "65": 10.82773, - "66": 10.84549, - "67": 10.82785, - "68": 10.82933, - "69": 10.81858, - "70": 10.82823, - "71": 10.82685, - "72": 10.80812, - "73": 10.8077, - "74": 10.80648, - "75": 10.81481, - "76": 10.81136, - "77": 10.80743, - "78": 10.79191, - "79": 10.80241, - "80": 10.78881, - "81": 10.79504, - "82": 10.79535, - "83": 10.78597, - "84": 10.7569, - "85": 10.76252, + "1": 10.90111, + "2": 10.8957, + "3": 10.90636, + "4": 10.90618, + "5": 10.91363, + "6": 10.89761, + "7": 10.89776, + "8": 10.91415, + "9": 10.89561, + "10": 10.9037, + "11": 10.90078, + "12": 10.91896, + "13": 10.88436, + "14": 10.8755, + "15": 10.91016, + "16": 10.90356, + "17": 10.88647, + "18": 10.89373, + "19": 10.89488, + "20": 10.89145, + "21": 10.8867, + "22": 10.90339, + "23": 10.9122, + "24": 10.88545, + "25": 10.90147, + "26": 10.88807, + "27": 10.89667, + "28": 10.88181, + "29": 10.88999, + "30": 10.91088, + "31": 10.89164, + "32": 10.88874, + "33": 10.89699, + "34": 10.87341, + "35": 10.89671, + "36": 10.90754, + "37": 10.87082, + "38": 10.87845, + "39": 10.8861, + "40": 10.89256, + "41": 10.88453, + "42": 10.89553, + "43": 10.88019, + "44": 10.88447, + "45": 10.88361, + "46": 10.88675, + "47": 10.88597, + "48": 10.8638, + "49": 10.8766, + "50": 10.88241, + "51": 10.89271, + "52": 10.87137, + "53": 10.85905, + "54": 10.87009, + "55": 10.86786, + "56": 10.87291, + "57": 10.84458, + "58": 10.85672, + "59": 10.84682, + "60": 10.84354, + "61": 10.86079, + "62": 10.85703, + "63": 10.86049, + "64": 10.83806, + "65": 10.82753, + "66": 10.84631, + "67": 10.82788, + "68": 10.83033, + "69": 10.81867, + "70": 10.82807, + "71": 10.82687, + "72": 10.80846, + "73": 10.80745, + "74": 10.8061, + "75": 10.81421, + "76": 10.81095, + "77": 10.80817, + "78": 10.79232, + "79": 10.80249, + "80": 10.78847, + "81": 10.79536, + "82": 10.79574, + "83": 10.78549, + "84": 10.75707, + "85": 10.76214, "86": 10.77729, - "87": 10.79571, - "88": 10.77474, - "89": 10.77531, - "90": 10.76553, - "91": 10.74039, - "92": 10.76056, - "93": 10.74732, - "94": 10.73435, - "95": 10.75251, - "96": 10.72334, - "97": 10.71517, - "98": 10.72635, - "99": 10.74623, - "100": 10.69589 + "87": 10.79586, + "88": 10.77534, + "89": 10.77602, + "90": 10.76492, + "91": 10.74145, + "92": 10.75946, + "93": 10.74599, + "94": 10.73392, + "95": 10.75202, + "96": 10.72408, + "97": 10.71554, + "98": 10.7261, + "99": 10.74663, + "100": 10.69609 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1166.0, - "2": 1181.0, - "3": 1369.0, - "4": 1274.0, - "5": 1227.0, - "6": 1240.0, - "7": 1282.0, - "8": 1206.0, - "9": 1161.0, - "10": 1142.0, - "11": 1186.0, - "12": 1297.0, + "1": 1133.0, + "2": 1120.0, + "3": 1360.0, + "4": 1289.0, + "5": 1206.0, + "6": 1218.0, + "7": 1344.0, + "8": 1181.0, + "9": 1206.0, + "10": 1139.0, + "11": 1252.0, + "12": 1218.0, "13": 1288.0, - "14": 1215.0, - "15": 1218.0, - "16": 1231.0, - "17": 1213.0, - "18": 1105.0, - "19": 1178.0, - "20": 1096.0, - "21": 1231.0, - "22": 1266.0, - "23": 1278.0, - "24": 1169.0, - "25": 1196.0, - "26": 1212.0, - "27": 1344.0, - "28": 1218.0, - "29": 1272.0, - "30": 1288.0, - "31": 1188.0, - "32": 1250.0, - "33": 1168.0, - "34": 1259.0, - "35": 1296.0, - "36": 1204.0, - "37": 1184.0, - "38": 1219.0, - "39": 1386.0, - "40": 1284.0, - "41": 1298.0, - "42": 1146.0, - "43": 1156.0, - "44": 1277.0, - "45": 1128.0, - "46": 1381.0, - "47": 1194.0, - "48": 1137.0, - "49": 1369.0, - "50": 1277.0, - "51": 1257.0, - "52": 1159.0, - "53": 1253.0, - "54": 1202.0, - "55": 1181.0, - "56": 1178.0, - "57": 1233.0, - "58": 1260.0, - "59": 1084.0, - "60": 1188.0, - "61": 1240.0, - "62": 1173.0, - "63": 1188.0, - "64": 1248.0, - "65": 1222.0, - "66": 1194.0, - "67": 1269.0, - "68": 1288.0, - "69": 1161.0, - "70": 1300.0, - "71": 1235.0, - "72": 1206.0, - "73": 1272.0, - "74": 1254.0, - "75": 1392.0, - "76": 1292.0, - "77": 1155.0, - "78": 1245.0, - "79": 1316.0, - "80": 1145.0, - "81": 1304.0, - "82": 1213.0, - "83": 1290.0, - "84": 1121.0, - "85": 1162.0, - "86": 1310.0, - "87": 1123.0, - "88": 1197.0, - "89": 1089.0, - "90": 1438.0, - "91": 1300.0, - "92": 1143.0, - "93": 1293.0, - "94": 1353.0, - "95": 1061.0, - "96": 1250.0, - "97": 1216.0, - "98": 1310.0, - "99": 1121.0, - "100": 1167.0 + "14": 1162.0, + "15": 1248.0, + "16": 1265.0, + "17": 1188.0, + "18": 1115.0, + "19": 1172.0, + "20": 1089.0, + "21": 1213.0, + "22": 1244.0, + "23": 1290.0, + "24": 1071.0, + "25": 1243.0, + "26": 1216.0, + "27": 1393.0, + "28": 1179.0, + "29": 1194.0, + "30": 1353.0, + "31": 1240.0, + "32": 1343.0, + "33": 1190.0, + "34": 1196.0, + "35": 1270.0, + "36": 1169.0, + "37": 1196.0, + "38": 1212.0, + "39": 1344.0, + "40": 1272.0, + "41": 1350.0, + "42": 1153.0, + "43": 1231.0, + "44": 1234.0, + "45": 1094.0, + "46": 1458.0, + "47": 1192.0, + "48": 1140.0, + "49": 1412.0, + "50": 1316.0, + "51": 1261.0, + "52": 1161.0, + "53": 1242.0, + "54": 1183.0, + "55": 1246.0, + "56": 1243.0, + "57": 1208.0, + "58": 1283.0, + "59": 1108.0, + "60": 1224.0, + "61": 1277.0, + "62": 1192.0, + "63": 1234.0, + "64": 1285.0, + "65": 1166.0, + "66": 1238.0, + "67": 1128.0, + "68": 1281.0, + "69": 1115.0, + "70": 1255.0, + "71": 1199.0, + "72": 1239.0, + "73": 1264.0, + "74": 1318.0, + "75": 1344.0, + "76": 1250.0, + "77": 1197.0, + "78": 1241.0, + "79": 1270.0, + "80": 1227.0, + "81": 1238.0, + "82": 1078.0, + "83": 1220.0, + "84": 1190.0, + "85": 1187.0, + "86": 1238.0, + "87": 1074.0, + "88": 1182.0, + "89": 1184.0, + "90": 1343.0, + "91": 1308.0, + "92": 1173.0, + "93": 1342.0, + "94": 1400.0, + "95": 1059.0, + "96": 1129.0, + "97": 1263.0, + "98": 1344.0, + "99": 1065.0, + "100": 1185.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1096817664.0, - "2": 1096807936.0, - "3": 1096783872.0, - "4": 1096764416.0, - "5": 1096815104.0, - "6": 1096799744.0, - "7": 1096722944.0, - "8": 1096760832.0, - "9": 1096762880.0, - "10": 1096796160.0, - "11": 1096770048.0, - "12": 1096778752.0, - "13": 1096783360.0, - "14": 1096783360.0, - "15": 1096762880.0, - "16": 1096748032.0, - "17": 1096756224.0, - "18": 1096758272.0, - "19": 1096789504.0, - "20": 1096754176.0, - "21": 1096804352.0, - "22": 1096830464.0, - "23": 1096802304.0, - "24": 1096719872.0, - "25": 1096800768.0, - "26": 1096830464.0, - "27": 1096781824.0, - "28": 1096736768.0, - "29": 1096810496.0, - "30": 1096745984.0, - "31": 1096750592.0, - "32": 1096758784.0, - "33": 1096764928.0, - "34": 1096742400.0, - "35": 1096730112.0, - "36": 1096793600.0, - "37": 1096743424.0, - "38": 1096801792.0, - "39": 1096810496.0, - "40": 1096754688.0, - "41": 1096777728.0, - "42": 1096720384.0, - "43": 1096785920.0, - "44": 1096770560.0, - "45": 1096745472.0, - "46": 1096808960.0, - "47": 1096766464.0, - "48": 1096794112.0, - "49": 1096755712.0, - "50": 1096802816.0, - "51": 1096785920.0, - "52": 1096758784.0, - "53": 1096707072.0, - "54": 1096805888.0, - "55": 1096795648.0, - "56": 1096794112.0, - "57": 1096855552.0, - "58": 1096790528.0, - "59": 1096795648.0, - "60": 1096774144.0, - "61": 1096743936.0, - "62": 1096733696.0, - "63": 1096740352.0, - "64": 1096771584.0, - "65": 1096778752.0, - "66": 1096792064.0, - "67": 1096775168.0, - "68": 1096774144.0, - "69": 1096738816.0, - "70": 1096822784.0, - "71": 1096799744.0, - "72": 1096799232.0, - "73": 1096731136.0, - "74": 1096769536.0, - "75": 1096816640.0, - "76": 1096772096.0, - "77": 1096817664.0, - "78": 1096759296.0, - "79": 1096826880.0, - "80": 1096746496.0, - "81": 1096800768.0, - "82": 1096747520.0, - "83": 1096810496.0, - "84": 1096782336.0, - "85": 1096786432.0, - "86": 1096771072.0, - "87": 1096785408.0, - "88": 1096768512.0, - "89": 1096757760.0, - "90": 1096779776.0, - "91": 1096765440.0, - "92": 1096763392.0, - "93": 1096793600.0, - "94": 1096767488.0, - "95": 1096791552.0, - "96": 1096782848.0, - "97": 1096751104.0, - "98": 1096788480.0, - "99": 1096806400.0, - "100": 1096837120.0 + "1": 1096719872.0, + "2": 1096710656.0, + "3": 1096685568.0, + "4": 1096666112.0, + "5": 1096716288.0, + "6": 1096698368.0, + "7": 1096625664.0, + "8": 1096664064.0, + "9": 1096666624.0, + "10": 1096697344.0, + "11": 1096671744.0, + "12": 1096680448.0, + "13": 1096684032.0, + "14": 1096687104.0, + "15": 1096663040.0, + "16": 1096648192.0, + "17": 1096657920.0, + "18": 1096658944.0, + "19": 1096689152.0, + "20": 1096654336.0, + "21": 1096707584.0, + "22": 1096734208.0, + "23": 1096705024.0, + "24": 1096621568.0, + "25": 1096701952.0, + "26": 1096734208.0, + "27": 1096685056.0, + "28": 1096639488.0, + "29": 1096713216.0, + "30": 1096646656.0, + "31": 1096652800.0, + "32": 1096659456.0, + "33": 1096667648.0, + "34": 1096645120.0, + "35": 1096631296.0, + "36": 1096694272.0, + "37": 1096644096.0, + "38": 1096703488.0, + "39": 1096710144.0, + "40": 1096657408.0, + "41": 1096678912.0, + "42": 1096621568.0, + "43": 1096688128.0, + "44": 1096673792.0, + "45": 1096646656.0, + "46": 1096709632.0, + "47": 1096666624.0, + "48": 1096695808.0, + "49": 1096656896.0, + "50": 1096704000.0, + "51": 1096687616.0, + "52": 1096661504.0, + "53": 1096610816.0, + "54": 1096705024.0, + "55": 1096696320.0, + "56": 1096693248.0, + "57": 1096753664.0, + "58": 1096691712.0, + "59": 1096695808.0, + "60": 1096675840.0, + "61": 1096647168.0, + "62": 1096636416.0, + "63": 1096645632.0, + "64": 1096672256.0, + "65": 1096679936.0, + "66": 1096693248.0, + "67": 1096674816.0, + "68": 1096677376.0, + "69": 1096642560.0, + "70": 1096725504.0, + "71": 1096706560.0, + "72": 1096703488.0, + "73": 1096632320.0, + "74": 1096673792.0, + "75": 1096717312.0, + "76": 1096673280.0, + "77": 1096714752.0, + "78": 1096664064.0, + "79": 1096729600.0, + "80": 1096649216.0, + "81": 1096703488.0, + "82": 1096649728.0, + "83": 1096710144.0, + "84": 1096683008.0, + "85": 1096688640.0, + "86": 1096672256.0, + "87": 1096687104.0, + "88": 1096668672.0, + "89": 1096658944.0, + "90": 1096683008.0, + "91": 1096666112.0, + "92": 1096664576.0, + "93": 1096695808.0, + "94": 1096671744.0, + "95": 1096692736.0, + "96": 1096684544.0, + "97": 1096652800.0, + "98": 1096691200.0, + "99": 1096708608.0, + "100": 1096738304.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3258557952.0, - "2": 3590636544.0, - "3": 3590636544.0, - "4": 3590636544.0, - "5": 3596905472.0, - "6": 3596905472.0, - "7": 3596905472.0, - "8": 3596905472.0, - "9": 3596905472.0, - "10": 3596905472.0, - "11": 3596905472.0, - "12": 3596905472.0, - "13": 3596905472.0, - "14": 3596905472.0, - "15": 3596905472.0, - "16": 3596905472.0, - "17": 3596905472.0, - "18": 3596905472.0, - "19": 3596905472.0, - "20": 3596905472.0, - "21": 3596905472.0, - "22": 3609437184.0, - "23": 3609437184.0, - "24": 3609437184.0, - "25": 3609437184.0, - "26": 3611321344.0, - "27": 3611321344.0, - "28": 3611321344.0, - "29": 3611321344.0, - "30": 3611321344.0, - "31": 3611321344.0, - "32": 3611321344.0, - "33": 3611321344.0, - "34": 3611321344.0, - "35": 3611321344.0, - "36": 3611321344.0, - "37": 3611321344.0, - "38": 3611321344.0, - "39": 3611321344.0, - "40": 3611321344.0, - "41": 3611321344.0, - "42": 3611321344.0, - "43": 3611321344.0, - "44": 3611321344.0, - "45": 3611321344.0, - "46": 3611321344.0, - "47": 3611321344.0, - "48": 3611321344.0, - "49": 3611321344.0, - "50": 3611321344.0, - "51": 3611321344.0, - "52": 3611321344.0, - "53": 3611321344.0, - "54": 3611321344.0, - "55": 3611321344.0, - "56": 3611321344.0, - "57": 3630413312.0, - "58": 3630413312.0, - "59": 3630413312.0, - "60": 3630413312.0, - "61": 3630413312.0, - "62": 3630413312.0, - "63": 3630413312.0, - "64": 3630413312.0, - "65": 3630413312.0, - "66": 3630413312.0, - "67": 3630413312.0, - "68": 3630413312.0, - "69": 3630413312.0, - "70": 3630413312.0, - "71": 3630413312.0, - "72": 3630413312.0, - "73": 3630413312.0, - "74": 3630413312.0, - "75": 3630413312.0, - "76": 3630413312.0, - "77": 3630413312.0, - "78": 3630413312.0, - "79": 3630413312.0, - "80": 3630413312.0, - "81": 3630413312.0, - "82": 3630413312.0, - "83": 3630413312.0, - "84": 3630413312.0, - "85": 3630413312.0, - "86": 3630413312.0, - "87": 3630413312.0, - "88": 3630413312.0, - "89": 3630413312.0, - "90": 3630413312.0, - "91": 3630413312.0, - "92": 3630413312.0, - "93": 3630413312.0, - "94": 3630413312.0, - "95": 3630413312.0, - "96": 3630413312.0, - "97": 3630413312.0, - "98": 3630413312.0, - "99": 3630413312.0, - "100": 3630413312.0 + "1": 3257109504.0, + "2": 3590822912.0, + "3": 3590822912.0, + "4": 3590822912.0, + "5": 3596996096.0, + "6": 3596996096.0, + "7": 3596996096.0, + "8": 3596996096.0, + "9": 3596996096.0, + "10": 3596996096.0, + "11": 3596996096.0, + "12": 3596996096.0, + "13": 3596996096.0, + "14": 3596996096.0, + "15": 3596996096.0, + "16": 3596996096.0, + "17": 3596996096.0, + "18": 3596996096.0, + "19": 3596996096.0, + "20": 3596996096.0, + "21": 3596996096.0, + "22": 3610441728.0, + "23": 3610441728.0, + "24": 3610441728.0, + "25": 3610441728.0, + "26": 3611658240.0, + "27": 3611658240.0, + "28": 3611658240.0, + "29": 3611658240.0, + "30": 3611658240.0, + "31": 3611658240.0, + "32": 3611658240.0, + "33": 3611658240.0, + "34": 3611658240.0, + "35": 3611658240.0, + "36": 3611658240.0, + "37": 3611658240.0, + "38": 3611658240.0, + "39": 3611658240.0, + "40": 3611658240.0, + "41": 3611658240.0, + "42": 3611658240.0, + "43": 3611658240.0, + "44": 3611658240.0, + "45": 3611658240.0, + "46": 3611658240.0, + "47": 3611658240.0, + "48": 3611658240.0, + "49": 3611658240.0, + "50": 3611658240.0, + "51": 3611658240.0, + "52": 3611658240.0, + "53": 3611658240.0, + "54": 3611658240.0, + "55": 3611658240.0, + "56": 3611658240.0, + "57": 3625479168.0, + "58": 3625479168.0, + "59": 3625479168.0, + "60": 3625479168.0, + "61": 3625479168.0, + "62": 3625479168.0, + "63": 3625479168.0, + "64": 3625479168.0, + "65": 3625479168.0, + "66": 3625479168.0, + "67": 3625479168.0, + "68": 3625479168.0, + "69": 3625479168.0, + "70": 3625479168.0, + "71": 3625479168.0, + "72": 3625479168.0, + "73": 3625479168.0, + "74": 3625479168.0, + "75": 3625479168.0, + "76": 3625479168.0, + "77": 3625479168.0, + "78": 3625479168.0, + "79": 3625479168.0, + "80": 3625479168.0, + "81": 3625479168.0, + "82": 3625479168.0, + "83": 3625479168.0, + "84": 3625479168.0, + "85": 3625479168.0, + "86": 3625479168.0, + "87": 3625479168.0, + "88": 3625479168.0, + "89": 3625479168.0, + "90": 3625479168.0, + "91": 3625479168.0, + "92": 3625479168.0, + "93": 3625479168.0, + "94": 3625479168.0, + "95": 3625479168.0, + "96": 3625479168.0, + "97": 3625479168.0, + "98": 3625479168.0, + "99": 3625479168.0, + "100": 3625479168.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 4.78938, - "3": 0.60373, - "4": 0.59311, - "5": 0.59195, - "6": 0.59273, - "7": 0.57837, - "8": 0.60657, - "9": 0.57308, - "10": 0.57962, - "11": 0.59121, - "12": 0.57012, - "13": 0.57956, - "14": 0.57252, - "15": 0.5778, - "16": 0.57556, - "17": 0.58331, - "18": 0.56468, - "19": 0.57301, - "20": 0.56966, - "21": 0.56425, - "22": 0.57413, - "23": 0.55995, - "24": 0.57786, - "25": 0.57348, - "26": 0.56472, - "27": 0.55511, - "28": 0.56315, - "29": 0.57013, - "30": 0.56578, - "31": 0.57204, - "32": 0.56786, - "33": 0.56996, - "34": 0.56576, - "35": 0.56723, - "36": 0.56311, - "37": 0.55743, - "38": 0.56454, - "39": 0.56626, - "40": 0.56817, - "41": 0.56308, - "42": 0.56431, - "43": 0.55892, - "44": 0.56373, - "45": 0.57067, - "46": 0.55855, - "47": 0.56537, - "48": 0.56161, - "49": 0.55823, - "50": 0.55897, - "51": 1.51215, - "52": 0.58969, - "53": 0.5794, - "54": 0.56179, - "55": 0.55753, - "56": 0.55763, - "57": 0.56517, - "58": 0.56733, - "59": 0.55665, - "60": 0.5759, - "61": 0.65482, - "62": 0.57041, - "63": 0.55863, - "64": 0.56279, - "65": 0.55663, - "66": 0.56047, - "67": 0.5619, - "68": 0.56148, - "69": 0.55887, - "70": 0.5592, - "71": 0.56258, - "72": 0.56253, - "73": 0.56437, - "74": 0.56609, - "75": 0.55638, - "76": 0.57034, - "77": 0.56328, - "78": 0.56443, - "79": 0.56674, - "80": 0.56048, - "81": 0.5663, - "82": 0.55975, - "83": 0.55919, - "84": 0.55883, - "85": 0.55711, - "86": 0.56565, - "87": 0.55683, - "88": 0.58115, - "89": 0.57514, - "90": 0.57865, - "91": 0.56708, - "92": 0.56269, - "93": 0.55817, - "94": 0.56214, - "95": 0.56258, - "96": 0.56495, - "97": 0.56018, - "98": 0.55666, - "99": 0.5647, - "100": 0.55567 + "2": 7.98917, + "3": 0.61774, + "4": 0.60327, + "5": 0.61118, + "6": 0.60341, + "7": 0.60198, + "8": 0.60804, + "9": 0.59243, + "10": 0.60275, + "11": 0.61409, + "12": 0.59478, + "13": 0.59416, + "14": 0.61157, + "15": 0.59528, + "16": 0.58201, + "17": 0.59897, + "18": 0.5747, + "19": 0.5852, + "20": 0.58745, + "21": 0.58032, + "22": 0.5811, + "23": 0.57991, + "24": 0.58907, + "25": 0.58791, + "26": 0.59379, + "27": 0.58561, + "28": 0.58974, + "29": 0.5884, + "30": 0.60742, + "31": 0.58261, + "32": 0.58956, + "33": 0.58555, + "34": 0.58583, + "35": 0.59496, + "36": 0.6018, + "37": 0.59609, + "38": 0.58356, + "39": 0.57802, + "40": 0.57925, + "41": 0.57195, + "42": 0.57595, + "43": 0.57774, + "44": 0.56745, + "45": 0.57916, + "46": 0.56772, + "47": 0.57299, + "48": 0.56576, + "49": 0.56807, + "50": 0.56685, + "51": 0.63208, + "52": 0.59958, + "53": 0.57872, + "54": 0.56971, + "55": 0.57134, + "56": 0.56955, + "57": 0.57154, + "58": 0.57832, + "59": 0.57163, + "60": 0.57009, + "61": 0.57588, + "62": 0.58149, + "63": 0.57169, + "64": 0.56388, + "65": 0.56972, + "66": 0.56979, + "67": 0.56465, + "68": 0.56683, + "69": 0.57178, + "70": 0.57356, + "71": 0.5728, + "72": 0.57218, + "73": 0.58671, + "74": 0.57786, + "75": 0.57202, + "76": 0.56452, + "77": 0.57511, + "78": 0.57141, + "79": 0.56707, + "80": 0.56945, + "81": 0.57006, + "82": 0.58157, + "83": 0.5684, + "84": 0.5733, + "85": 0.5679, + "86": 0.57571, + "87": 0.57676, + "88": 0.57866, + "89": 0.57356, + "90": 0.57899, + "91": 0.57243, + "92": 0.6007, + "93": 0.59639, + "94": 0.58713, + "95": 0.59701, + "96": 0.60178, + "97": 0.57955, + "98": 0.5712, + "99": 0.57566, + "100": 0.58236 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json index 467d686c187..b441416be8b 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.93025, - "2": 10.92269, - "3": 10.9252, - "4": 10.91732, - "5": 10.92716, - "6": 10.92029, - "7": 10.91689, - "8": 10.91796, - "9": 10.93832, - "10": 10.9236, - "11": 10.91613, - "12": 10.92602, - "13": 10.93472, - "14": 10.91527, - "15": 10.92824, - "16": 10.91861, - "17": 10.9271, - "18": 10.91296, - "19": 10.93023, - "20": 10.90016, - "21": 10.93966, - "22": 10.92562, - "23": 10.93028, - "24": 10.90543, - "25": 10.91525, - "26": 10.9253, - "27": 10.92741, - "28": 10.92037, - "29": 10.92588, + "1": 10.93064, + "2": 10.92264, + "3": 10.92453, + "4": 10.91706, + "5": 10.92766, + "6": 10.92069, + "7": 10.9166, + "8": 10.91808, + "9": 10.93792, + "10": 10.92356, + "11": 10.91567, + "12": 10.92593, + "13": 10.93441, + "14": 10.91528, + "15": 10.92879, + "16": 10.9189, + "17": 10.92699, + "18": 10.91334, + "19": 10.93068, + "20": 10.90002, + "21": 10.9404, + "22": 10.92589, + "23": 10.93025, + "24": 10.90535, + "25": 10.91492, + "26": 10.92587, + "27": 10.9279, + "28": 10.91992, + "29": 10.92593, "30": 10.91378, - "31": 10.91042, - "32": 10.91855, - "33": 10.91979, - "34": 10.90066, - "35": 10.91944, - "36": 10.90776, - "37": 10.91233, - "38": 10.92343, - "39": 10.91274, - "40": 10.91036, - "41": 10.91294, - "42": 10.90908, - "43": 10.90588, - "44": 10.90751, - "45": 10.89747, - "46": 10.90732, - "47": 10.90263, - "48": 10.88416, - "49": 10.90034, - "50": 10.89852, - "51": 10.89775, - "52": 10.9041, - "53": 10.90148, - "54": 10.88746, - "55": 10.88695, - "56": 10.88936, - "57": 10.88718, - "58": 10.88857, - "59": 10.88171, - "60": 10.87361, - "61": 10.8735, - "62": 10.86885, - "63": 10.87575, - "64": 10.85989, - "65": 10.86121, - "66": 10.85651, - "67": 10.86113, - "68": 10.86021, - "69": 10.85001, - "70": 10.86693, - "71": 10.84848, - "72": 10.84069, - "73": 10.84796, - "74": 10.83901, - "75": 10.83614, - "76": 10.83454, - "77": 10.82879, - "78": 10.82267, - "79": 10.82851, - "80": 10.82921, - "81": 10.81793, - "82": 10.8169, - "83": 10.80474, - "84": 10.78276, - "85": 10.78362, - "86": 10.80088, - "87": 10.7953, - "88": 10.79578, - "89": 10.77732, - "90": 10.78292, - "91": 10.78991, - "92": 10.7824, - "93": 10.75549, - "94": 10.76106, - "95": 10.7731, - "96": 10.73578, - "97": 10.74142, - "98": 10.74708, - "99": 10.76219, - "100": 10.74211 + "31": 10.9109, + "32": 10.91882, + "33": 10.92004, + "34": 10.90075, + "35": 10.91879, + "36": 10.90792, + "37": 10.91301, + "38": 10.92359, + "39": 10.9122, + "40": 10.91099, + "41": 10.91319, + "42": 10.90912, + "43": 10.90596, + "44": 10.90688, + "45": 10.89722, + "46": 10.90681, + "47": 10.90148, + "48": 10.8843, + "49": 10.90016, + "50": 10.89916, + "51": 10.89786, + "52": 10.90405, + "53": 10.90069, + "54": 10.88764, + "55": 10.88607, + "56": 10.88893, + "57": 10.88739, + "58": 10.88856, + "59": 10.88187, + "60": 10.87372, + "61": 10.87299, + "62": 10.86808, + "63": 10.87591, + "64": 10.85915, + "65": 10.86096, + "66": 10.85725, + "67": 10.86131, + "68": 10.86071, + "69": 10.85024, + "70": 10.86702, + "71": 10.84888, + "72": 10.83992, + "73": 10.84898, + "74": 10.839, + "75": 10.83634, + "76": 10.83445, + "77": 10.82965, + "78": 10.82317, + "79": 10.82842, + "80": 10.82931, + "81": 10.81726, + "82": 10.81715, + "83": 10.80494, + "84": 10.78359, + "85": 10.78354, + "86": 10.80074, + "87": 10.79543, + "88": 10.79476, + "89": 10.77699, + "90": 10.78295, + "91": 10.79036, + "92": 10.78271, + "93": 10.75468, + "94": 10.7609, + "95": 10.77286, + "96": 10.73628, + "97": 10.74115, + "98": 10.74693, + "99": 10.76285, + "100": 10.74247 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 48601.0, - "2": 50656.0, - "3": 49241.0, - "4": 48921.0, - "5": 50418.0, - "6": 49904.0, - "7": 51873.0, - "8": 50118.0, - "9": 50874.0, - "10": 52037.0, - "11": 50176.0, - "12": 48637.0, - "13": 52081.0, - "14": 50045.0, - "15": 48440.0, - "16": 49755.0, - "17": 49654.0, - "18": 51941.0, - "19": 51305.0, - "20": 52452.0, - "21": 50160.0, - "22": 49759.0, - "23": 53668.0, - "24": 51479.0, - "25": 48721.0, - "26": 52060.0, - "27": 51901.0, - "28": 48375.0, - "29": 50453.0, - "30": 51324.0, - "31": 52343.0, - "32": 49490.0, - "33": 51963.0, - "34": 50282.0, - "35": 51151.0, - "36": 51128.0, - "37": 49489.0, - "38": 50866.0, - "39": 50922.0, - "40": 50972.0, - "41": 48289.0, - "42": 50130.0, - "43": 51143.0, - "44": 47593.0, - "45": 54555.0, - "46": 48428.0, - "47": 50067.0, - "48": 50925.0, - "49": 54379.0, - "50": 51185.0, - "51": 48440.0, - "52": 50898.0, - "53": 49761.0, - "54": 48987.0, - "55": 49733.0, - "56": 48541.0, - "57": 48439.0, - "58": 52836.0, - "59": 50186.0, - "60": 49645.0, - "61": 47579.0, - "62": 50843.0, - "63": 48799.0, - "64": 56194.0, - "65": 51032.0, - "66": 49352.0, - "67": 52039.0, - "68": 49477.0, - "69": 54578.0, - "70": 49750.0, - "71": 49318.0, - "72": 52184.0, - "73": 53016.0, - "74": 49953.0, - "75": 50635.0, - "76": 51334.0, - "77": 50040.0, - "78": 49918.0, - "79": 48849.0, - "80": 51988.0, - "81": 50001.0, - "82": 46662.0, - "83": 52909.0, - "84": 48924.0, - "85": 50721.0, - "86": 49065.0, - "87": 46136.0, - "88": 48331.0, - "89": 48705.0, - "90": 52429.0, - "91": 50491.0, - "92": 50216.0, - "93": 50541.0, - "94": 51146.0, - "95": 48468.0, - "96": 49648.0, - "97": 49767.0, - "98": 48502.0, - "99": 49428.0, - "100": 47072.0 + "1": 49180.0, + "2": 50476.0, + "3": 49152.0, + "4": 49093.0, + "5": 49913.0, + "6": 49695.0, + "7": 51527.0, + "8": 49236.0, + "9": 50963.0, + "10": 51988.0, + "11": 50179.0, + "12": 48660.0, + "13": 52142.0, + "14": 50135.0, + "15": 47947.0, + "16": 49733.0, + "17": 49715.0, + "18": 51893.0, + "19": 50900.0, + "20": 52897.0, + "21": 50249.0, + "22": 49494.0, + "23": 53440.0, + "24": 51476.0, + "25": 48836.0, + "26": 52057.0, + "27": 51764.0, + "28": 48472.0, + "29": 50091.0, + "30": 51160.0, + "31": 52934.0, + "32": 49271.0, + "33": 52492.0, + "34": 49785.0, + "35": 51389.0, + "36": 50759.0, + "37": 49447.0, + "38": 50917.0, + "39": 50977.0, + "40": 51303.0, + "41": 47884.0, + "42": 50199.0, + "43": 51264.0, + "44": 48123.0, + "45": 54319.0, + "46": 47721.0, + "47": 50709.0, + "48": 50505.0, + "49": 54725.0, + "50": 50759.0, + "51": 48818.0, + "52": 50484.0, + "53": 49662.0, + "54": 49308.0, + "55": 49348.0, + "56": 48679.0, + "57": 48551.0, + "58": 52274.0, + "59": 49618.0, + "60": 49684.0, + "61": 47669.0, + "62": 50900.0, + "63": 48419.0, + "64": 55920.0, + "65": 51019.0, + "66": 49471.0, + "67": 51576.0, + "68": 49653.0, + "69": 54382.0, + "70": 50016.0, + "71": 49561.0, + "72": 51901.0, + "73": 52536.0, + "74": 49702.0, + "75": 50683.0, + "76": 51554.0, + "77": 49757.0, + "78": 49608.0, + "79": 49124.0, + "80": 52318.0, + "81": 49552.0, + "82": 46737.0, + "83": 52475.0, + "84": 48461.0, + "85": 50083.0, + "86": 49539.0, + "87": 46565.0, + "88": 48397.0, + "89": 48861.0, + "90": 52531.0, + "91": 50128.0, + "92": 50276.0, + "93": 50731.0, + "94": 51663.0, + "95": 48703.0, + "96": 49914.0, + "97": 49912.0, + "98": 48976.0, + "99": 48894.0, + "100": 47024.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1257367552.0, - "2": 1257389056.0, + "1": 1257369088.0, + "2": 1257388032.0, "3": 1257372160.0, - "4": 1257369600.0, + "4": 1257370112.0, "5": 1257382400.0, - "6": 1257367040.0, - "7": 1257362944.0, - "8": 1257389056.0, - "9": 1257339904.0, - "10": 1257339392.0, - "11": 1257327616.0, - "12": 1257354240.0, - "13": 1257356800.0, - "14": 1257344000.0, - "15": 1257376256.0, - "16": 1257363968.0, - "17": 1257374208.0, + "6": 1257369600.0, + "7": 1257363456.0, + "8": 1257391104.0, + "9": 1257341440.0, + "10": 1257339904.0, + "11": 1257326080.0, + "12": 1257356288.0, + "13": 1257354240.0, + "14": 1257347584.0, + "15": 1257373184.0, + "16": 1257364480.0, + "17": 1257375232.0, "18": 1257352704.0, - "19": 1257348096.0, + "19": 1257348608.0, "20": 1257361408.0, "21": 1257383424.0, - "22": 1257317376.0, - "23": 1257361408.0, - "24": 1257375232.0, - "25": 1257339904.0, - "26": 1257375232.0, - "27": 1257379328.0, - "28": 1257344512.0, - "29": 1257361920.0, + "22": 1257318400.0, + "23": 1257361920.0, + "24": 1257377280.0, + "25": 1257337344.0, + "26": 1257374208.0, + "27": 1257379840.0, + "28": 1257345024.0, + "29": 1257360896.0, "30": 1257371136.0, - "31": 1257334272.0, - "32": 1257356288.0, - "33": 1257341440.0, + "31": 1257333760.0, + "32": 1257354752.0, + "33": 1257340928.0, "34": 1257382400.0, - "35": 1257312256.0, - "36": 1257382912.0, + "35": 1257311232.0, + "36": 1257384960.0, "37": 1257352704.0, - "38": 1257382400.0, - "39": 1257344000.0, + "38": 1257378304.0, + "39": 1257342464.0, "40": 1257394176.0, "41": 1257356800.0, - "42": 1257335808.0, - "43": 1257340416.0, - "44": 1257348608.0, - "45": 1257361920.0, - "46": 1257313792.0, + "42": 1257336832.0, + "43": 1257338368.0, + "44": 1257350656.0, + "45": 1257364480.0, + "46": 1257312256.0, "47": 1257336832.0, - "48": 1257327616.0, - "49": 1257357824.0, - "50": 1257340928.0, - "51": 1257310208.0, - "52": 1257357824.0, - "53": 1257312768.0, - "54": 1257373696.0, - "55": 1257321472.0, + "48": 1257326592.0, + "49": 1257357312.0, + "50": 1257341440.0, + "51": 1257309696.0, + "52": 1257357312.0, + "53": 1257314304.0, + "54": 1257373184.0, + "55": 1257322496.0, "56": 1257371136.0, - "57": 1257341952.0, - "58": 1257348096.0, - "59": 1257347584.0, - "60": 1257311744.0, + "57": 1257340928.0, + "58": 1257348608.0, + "59": 1257349120.0, + "60": 1257313792.0, "61": 1257295360.0, - "62": 1257329152.0, - "63": 1257332224.0, + "62": 1257327104.0, + "63": 1257333248.0, "64": 1257348608.0, - "65": 1257352192.0, - "66": 1257273856.0, + "65": 1257353216.0, + "66": 1257275392.0, "67": 1257360384.0, - "68": 1257308160.0, - "69": 1257333248.0, - "70": 1257301504.0, - "71": 1257320960.0, - "72": 1257318400.0, - "73": 1257319936.0, - "74": 1257283584.0, - "75": 1257308160.0, - "76": 1257313280.0, + "68": 1257307136.0, + "69": 1257333760.0, + "70": 1257299968.0, + "71": 1257321984.0, + "72": 1257320448.0, + "73": 1257316864.0, + "74": 1257284608.0, + "75": 1257309696.0, + "76": 1257312256.0, "77": 1257349632.0, - "78": 1257342976.0, - "79": 1257314304.0, - "80": 1257260032.0, - "81": 1257303552.0, - "82": 1257275392.0, + "78": 1257344512.0, + "79": 1257313280.0, + "80": 1257257984.0, + "81": 1257302528.0, + "82": 1257277952.0, "83": 1257269248.0, - "84": 1257310720.0, + "84": 1257312768.0, "85": 1257284096.0, - "86": 1257277952.0, - "87": 1257269248.0, - "88": 1257281024.0, - "89": 1257268224.0, - "90": 1257305600.0, - "91": 1257270784.0, - "92": 1257271808.0, - "93": 1257287680.0, - "94": 1257270272.0, - "95": 1257265152.0, - "96": 1257272320.0, - "97": 1257265152.0, + "86": 1257276928.0, + "87": 1257267712.0, + "88": 1257282560.0, + "89": 1257268736.0, + "90": 1257306112.0, + "91": 1257267712.0, + "92": 1257269760.0, + "93": 1257285120.0, + "94": 1257268736.0, + "95": 1257267712.0, + "96": 1257271808.0, + "97": 1257264640.0, "98": 1257313792.0, "99": 1257251328.0, - "100": 1257255936.0 + "100": 1257255424.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3479111680.0, - "2": 3875777024.0, - "3": 3875777024.0, - "4": 3875777024.0, - "5": 3875777024.0, - "6": 3875777024.0, - "7": 3875777024.0, - "8": 3875777024.0, - "9": 3875777024.0, - "10": 3875777024.0, - "11": 3875777024.0, - "12": 3875777024.0, - "13": 3875777024.0, - "14": 3875777024.0, - "15": 3875777024.0, - "16": 3875777024.0, - "17": 3875777024.0, - "18": 3875777024.0, - "19": 3875777024.0, - "20": 3875777024.0, - "21": 3875777024.0, - "22": 3875777024.0, - "23": 3875777024.0, - "24": 3875777024.0, - "25": 3875777024.0, - "26": 3875777024.0, - "27": 3875777024.0, - "28": 3875777024.0, - "29": 3875777024.0, - "30": 3875777024.0, - "31": 3875777024.0, - "32": 3875777024.0, - "33": 3875777024.0, - "34": 3875777024.0, - "35": 3875777024.0, - "36": 3875777024.0, - "37": 3875777024.0, - "38": 3875777024.0, - "39": 3875777024.0, - "40": 3875777024.0, - "41": 3875777024.0, - "42": 3875777024.0, - "43": 3875777024.0, - "44": 3875777024.0, - "45": 3875777024.0, - "46": 3875777024.0, - "47": 3875777024.0, - "48": 3875777024.0, - "49": 3875777024.0, - "50": 3875777024.0, - "51": 3875777024.0, - "52": 3875777024.0, - "53": 3875777024.0, - "54": 3875777024.0, - "55": 3875777024.0, - "56": 3875777024.0, - "57": 3875777024.0, - "58": 3875777024.0, - "59": 3875777024.0, - "60": 3875777024.0, - "61": 3875777024.0, - "62": 3875777024.0, - "63": 3875777024.0, - "64": 3875777024.0, - "65": 3875777024.0, - "66": 3875777024.0, - "67": 3875777024.0, - "68": 3875777024.0, - "69": 3875777024.0, - "70": 3875777024.0, - "71": 3875777024.0, - "72": 3875777024.0, - "73": 3875777024.0, - "74": 3875777024.0, - "75": 3875777024.0, - "76": 3875777024.0, - "77": 3875777024.0, - "78": 3875777024.0, - "79": 3875777024.0, - "80": 3875777024.0, - "81": 3875777024.0, - "82": 3875777024.0, - "83": 3875777024.0, - "84": 3875777024.0, - "85": 3875777024.0, - "86": 3875777024.0, - "87": 3875777024.0, - "88": 3875777024.0, - "89": 3875777024.0, - "90": 3875777024.0, - "91": 3875777024.0, - "92": 3875777024.0, - "93": 3875777024.0, - "94": 3875777024.0, - "95": 3875777024.0, - "96": 3875777024.0, - "97": 3875777024.0, - "98": 3875777024.0, - "99": 3875777024.0, - "100": 3875777024.0 + "1": 3479515648.0, + "2": 3875144192.0, + "3": 3875144192.0, + "4": 3875144192.0, + "5": 3875144192.0, + "6": 3875144192.0, + "7": 3875144192.0, + "8": 3875144192.0, + "9": 3875144192.0, + "10": 3875144192.0, + "11": 3875144192.0, + "12": 3875144192.0, + "13": 3875144192.0, + "14": 3875144192.0, + "15": 3875144192.0, + "16": 3875144192.0, + "17": 3875144192.0, + "18": 3875144192.0, + "19": 3875144192.0, + "20": 3875144192.0, + "21": 3875144192.0, + "22": 3875144192.0, + "23": 3875144192.0, + "24": 3875144192.0, + "25": 3875144192.0, + "26": 3875144192.0, + "27": 3875144192.0, + "28": 3875144192.0, + "29": 3875144192.0, + "30": 3875144192.0, + "31": 3875144192.0, + "32": 3875144192.0, + "33": 3875144192.0, + "34": 3875144192.0, + "35": 3875144192.0, + "36": 3875144192.0, + "37": 3875144192.0, + "38": 3875144192.0, + "39": 3875144192.0, + "40": 3875144192.0, + "41": 3875144192.0, + "42": 3875144192.0, + "43": 3875144192.0, + "44": 3875144192.0, + "45": 3875144192.0, + "46": 3875144192.0, + "47": 3875144192.0, + "48": 3875144192.0, + "49": 3875144192.0, + "50": 3875144192.0, + "51": 3875144192.0, + "52": 3875144192.0, + "53": 3875144192.0, + "54": 3875144192.0, + "55": 3875144192.0, + "56": 3875144192.0, + "57": 3875144192.0, + "58": 3875144192.0, + "59": 3875144192.0, + "60": 3875144192.0, + "61": 3875144192.0, + "62": 3875144192.0, + "63": 3875144192.0, + "64": 3875144192.0, + "65": 3875144192.0, + "66": 3875144192.0, + "67": 3875144192.0, + "68": 3875144192.0, + "69": 3875144192.0, + "70": 3875144192.0, + "71": 3875144192.0, + "72": 3875144192.0, + "73": 3875144192.0, + "74": 3875144192.0, + "75": 3875144192.0, + "76": 3875144192.0, + "77": 3875144192.0, + "78": 3875144192.0, + "79": 3875144192.0, + "80": 3875144192.0, + "81": 3875144192.0, + "82": 3875144192.0, + "83": 3875144192.0, + "84": 3875144192.0, + "85": 3875144192.0, + "86": 3875144192.0, + "87": 3875144192.0, + "88": 3875144192.0, + "89": 3875144192.0, + "90": 3875144192.0, + "91": 3875144192.0, + "92": 3875144192.0, + "93": 3875144192.0, + "94": 3875144192.0, + "95": 3875144192.0, + "96": 3875144192.0, + "97": 3875144192.0, + "98": 3875144192.0, + "99": 3875144192.0, + "100": 3875144192.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.74055, - "3": 0.71481, - "4": 0.71002, - "5": 0.70023, - "6": 0.80353, - "7": 1.08069, - "8": 0.82966, - "9": 0.68616, - "10": 0.69352, - "11": 1.3763, - "12": 0.67456, - "13": 0.68103, - "14": 0.68088, - "15": 0.68111, - "16": 0.6849, - "17": 0.6821, - "18": 0.67628, - "19": 0.67892, - "20": 0.67114, - "21": 0.6794, - "22": 0.67588, - "23": 0.66658, - "24": 0.68384, - "25": 0.67495, - "26": 0.67535, - "27": 0.67941, - "28": 0.67459, - "29": 0.67946, - "30": 0.6735, - "31": 0.67923, - "32": 0.67654, - "33": 0.66445, - "34": 0.67321, - "35": 0.67477, - "36": 0.66866, - "37": 0.66017, - "38": 0.67265, - "39": 0.67363, - "40": 0.67488, - "41": 0.67745, - "42": 0.67349, - "43": 0.6743, - "44": 0.67482, - "45": 0.68638, - "46": 0.67075, - "47": 0.67307, - "48": 0.67068, - "49": 0.6716, - "50": 0.67375, - "51": 0.75782, - "52": 0.70259, - "53": 0.68514, - "54": 0.66848, - "55": 0.68164, - "56": 0.66032, - "57": 0.66699, - "58": 0.67902, - "59": 0.66589, - "60": 0.66725, - "61": 0.66884, - "62": 0.69066, - "63": 0.66959, - "64": 0.67724, - "65": 0.68353, - "66": 0.68327, - "67": 0.6822, - "68": 0.6836, - "69": 0.68627, - "70": 0.74292, - "71": 0.66905, - "72": 0.67629, - "73": 0.67447, - "74": 0.66723, - "75": 0.68207, - "76": 0.66991, - "77": 0.65869, - "78": 1.02625, - "79": 0.66065, - "80": 0.67113, - "81": 0.66751, - "82": 0.78247, - "83": 0.73645, - "84": 0.67068, - "85": 0.66462, - "86": 0.66825, - "87": 0.671, - "88": 0.66284, - "89": 0.66802, - "90": 0.66231, - "91": 0.66494, - "92": 0.67077, - "93": 0.66016, - "94": 0.66796, - "95": 0.66874, - "96": 0.66957, - "97": 0.66025, - "98": 0.65909, - "99": 0.70097, - "100": 0.68106 + "2": 5.06008, + "3": 0.62212, + "4": 0.59439, + "5": 0.59583, + "6": 0.60184, + "7": 0.68944, + "8": 0.64314, + "9": 0.6749, + "10": 0.67461, + "11": 1.42804, + "12": 0.58165, + "13": 0.59318, + "14": 0.79281, + "15": 0.58563, + "16": 0.70911, + "17": 0.58841, + "18": 0.93635, + "19": 0.58466, + "20": 0.57567, + "21": 0.75501, + "22": 0.7513, + "23": 0.57466, + "24": 0.59062, + "25": 0.57249, + "26": 0.57164, + "27": 0.57991, + "28": 0.58345, + "29": 0.58425, + "30": 0.57143, + "31": 0.57751, + "32": 0.63679, + "33": 0.57343, + "34": 0.692, + "35": 0.576, + "36": 0.57893, + "37": 0.57805, + "38": 0.57591, + "39": 0.56481, + "40": 0.57011, + "41": 0.58034, + "42": 0.5834, + "43": 0.58412, + "44": 0.57894, + "45": 0.64352, + "46": 0.5782, + "47": 0.57996, + "48": 0.83465, + "49": 0.78578, + "50": 0.63159, + "51": 0.64775, + "52": 0.60094, + "53": 0.57927, + "54": 0.57978, + "55": 0.57904, + "56": 0.75419, + "57": 0.5724, + "58": 0.58029, + "59": 0.61934, + "60": 0.58073, + "61": 0.57339, + "62": 0.83927, + "63": 0.60116, + "64": 1.07001, + "65": 0.57078, + "66": 0.69353, + "67": 0.64965, + "68": 0.57507, + "69": 0.58624, + "70": 0.57896, + "71": 0.57264, + "72": 0.58547, + "73": 0.58352, + "74": 0.57325, + "75": 0.57046, + "76": 0.57555, + "77": 0.57548, + "78": 1.24371, + "79": 0.58474, + "80": 0.57365, + "81": 0.61678, + "82": 0.65707, + "83": 0.85963, + "84": 0.57211, + "85": 0.58096, + "86": 0.57084, + "87": 0.89269, + "88": 0.8608, + "89": 0.75853, + "90": 0.57419, + "91": 0.57351, + "92": 0.9021, + "93": 0.57971, + "94": 0.58609, + "95": 0.91409, + "96": 0.59088, + "97": 0.6732, + "98": 0.7339, + "99": 1.99078, + "100": 0.57381 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 2dc643a10f8..30c08fdc1d1 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -4,106 +4,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.92183, - "2": 10.9077, - "3": 10.90928, - "4": 10.9197, - "5": 10.90609, - "6": 10.91532, - "7": 10.92252, - "8": 10.90995, - "9": 10.91161, - "10": 10.90689, - "11": 10.88851, - "12": 10.90503, - "13": 10.89514, - "14": 10.89598, - "15": 10.87477, - "16": 10.86137, - "17": 10.86454, - "18": 10.85029, - "19": 10.85483, - "20": 10.78398, - "21": 10.77763, - "22": 10.75638, - "23": 10.74498, - "24": 10.71724, - "25": 10.71549, - "26": 10.71018, - "27": 10.67229, - "28": 10.60467, - "29": 10.57062, - "30": 10.54185, - "31": 10.54427, - "32": 10.53161, - "33": 10.49774, - "34": 10.47495, + "1": 10.92169, + "2": 10.90762, + "3": 10.90909, + "4": 10.91955, + "5": 10.90573, + "6": 10.91571, + "7": 10.92287, + "8": 10.90904, + "9": 10.91203, + "10": 10.90702, + "11": 10.88873, + "12": 10.90475, + "13": 10.89504, + "14": 10.89573, + "15": 10.87518, + "16": 10.8614, + "17": 10.86441, + "18": 10.85053, + "19": 10.85499, + "20": 10.78455, + "21": 10.7775, + "22": 10.75611, + "23": 10.74483, + "24": 10.71735, + "25": 10.71547, + "26": 10.71001, + "27": 10.67234, + "28": 10.60468, + "29": 10.57052, + "30": 10.54195, + "31": 10.54485, + "32": 10.53174, + "33": 10.49752, + "34": 10.47464, "35": 10.47083, - "36": 10.44074, - "37": 10.41615, - "38": 10.41757, - "39": 10.38481, - "40": 10.36819, - "41": 10.34676, - "42": 10.33402, - "43": 10.31355, - "44": 10.28912, - "45": 10.29322, - "46": 10.26555, - "47": 10.24535, - "48": 10.19867, - "49": 10.19508, + "36": 10.44096, + "37": 10.41642, + "38": 10.41727, + "39": 10.38507, + "40": 10.36831, + "41": 10.34679, + "42": 10.33436, + "43": 10.31332, + "44": 10.28897, + "45": 10.29332, + "46": 10.26565, + "47": 10.24514, + "48": 10.19831, + "49": 10.19477, "50": 10.1961, - "51": 10.20234, - "52": 10.15458, - "53": 10.16274, - "54": 10.12749, - "55": 10.09557, - "56": 10.1248, - "57": 10.11363, - "58": 10.12436, - "59": 10.07391, - "60": 10.0937, - "61": 10.04301, - "62": 10.0162, - "63": 10.08729, - "64": 10.03767, + "51": 10.20232, + "52": 10.15459, + "53": 10.16262, + "54": 10.12729, + "55": 10.09552, + "56": 10.12465, + "57": 10.11344, + "58": 10.12412, + "59": 10.07409, + "60": 10.09369, + "61": 10.04308, + "62": 10.0163, + "63": 10.08727, + "64": 10.03778, "65": 10.01755, - "66": 10.04075, + "66": 10.04058, "67": 10.01819, - "68": 9.97999, - "69": 10.00344, - "70": 9.98385, - "71": 10.01034, - "72": 9.99152, - "73": 9.98086, - "74": 9.96234, - "75": 9.94256, - "76": 9.97138, - "77": 9.96086, - "78": 9.91565, - "79": 9.92236, - "80": 9.93093, - "81": 9.9588, - "82": 9.89549, - "83": 9.8631, - "84": 9.79593, - "85": 9.78814, + "68": 9.9801, + "69": 10.00352, + "70": 9.98393, + "71": 10.01032, + "72": 9.99173, + "73": 9.98076, + "74": 9.9623, + "75": 9.94249, + "76": 9.97148, + "77": 9.96097, + "78": 9.91558, + "79": 9.92241, + "80": 9.93107, + "81": 9.95879, + "82": 9.89539, + "83": 9.86306, + "84": 9.79603, + "85": 9.78813, "86": 9.88181, "87": 9.90776, "88": 9.88258, - "89": 9.82994, - "90": 9.82197, - "91": 9.82939, - "92": 9.82025, - "93": 9.75994, - "94": 9.83142, - "95": 9.82809, - "96": 9.80747, - "97": 9.75381, - "98": 9.78418, - "99": 9.82603, - "100": 9.71824 + "89": 9.82999, + "90": 9.82224, + "91": 9.82944, + "92": 9.82018, + "93": 9.76029, + "94": 9.83143, + "95": 9.82804, + "96": 9.80736, + "97": 9.75384, + "98": 9.78406, + "99": 9.82607, + "100": 9.7184 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2443.0, - "2": 2426.0, - "3": 2525.0, - "4": 2507.0, - "5": 2396.0, - "6": 2390.0, - "7": 2713.0, - "8": 2450.0, - "9": 2503.0, - "10": 2522.0, - "11": 2501.0, - "12": 2415.0, - "13": 2653.0, - "14": 2579.0, - "15": 2431.0, - "16": 2402.0, - "17": 2493.0, - "18": 2552.0, - "19": 2604.0, - "20": 2483.0, - "21": 2598.0, - "22": 2493.0, - "23": 2503.0, - "24": 2584.0, - "25": 2435.0, - "26": 2468.0, - "27": 2494.0, - "28": 2636.0, - "29": 2684.0, - "30": 2519.0, - "31": 2839.0, - "32": 2808.0, - "33": 6456.0, - "34": 2874.0, - "35": 527564.0, - "36": 527663.0, - "37": 26089.0, - "38": 527761.0, - "39": 527729.0, - "40": 527902.0, - "41": 31694.0, - "42": 527567.0, - "43": 528026.0, - "44": 527859.0, - "45": 30402.0, - "46": 528061.0, - "47": 528003.0, - "48": 528169.0, - "49": 528568.0, - "50": 550768.0, - "51": 531413.0, - "52": 533217.0, - "53": 1059324.0, - "54": 554963.0, - "55": 554219.0, - "56": 558038.0, - "57": 1052795.0, - "58": 1078748.0, - "59": 558355.0, - "60": 582120.0, - "61": 587284.0, - "62": 1578182.0, - "63": 557832.0, - "64": 557805.0, - "65": 1578376.0, - "66": 1055033.0, - "67": 1085742.0, - "68": 1053875.0, - "69": 561563.0, - "70": 531055.0, - "71": 1054821.0, - "72": 550583.0, - "73": 1578398.0, - "74": 1075460.0, - "75": 557948.0, - "76": 1578104.0, - "77": 550589.0, - "78": 556345.0, - "79": 1077900.0, - "80": 578114.0, - "81": 1059016.0, - "82": 1055102.0, - "83": 1577764.0, - "84": 575310.0, - "85": 556803.0, - "86": 554852.0, - "87": 575521.0, - "88": 559536.0, - "89": 1053598.0, - "90": 1578544.0, - "91": 1053540.0, - "92": 1053497.0, - "93": 35286.0, - "94": 1053482.0, - "95": 530157.0, - "96": 1055299.0, - "97": 1054056.0, - "98": 1054267.0, - "99": 577154.0, - "100": 1578785.0 + "1": 2468.0, + "2": 2401.0, + "3": 2493.0, + "4": 2505.0, + "5": 2492.0, + "6": 2441.0, + "7": 2647.0, + "8": 2461.0, + "9": 2625.0, + "10": 2517.0, + "11": 2523.0, + "12": 2365.0, + "13": 2673.0, + "14": 2542.0, + "15": 2403.0, + "16": 2464.0, + "17": 2630.0, + "18": 2450.0, + "19": 2642.0, + "20": 2426.0, + "21": 2654.0, + "22": 2624.0, + "23": 2472.0, + "24": 2570.0, + "25": 2385.0, + "26": 2633.0, + "27": 2532.0, + "28": 2541.0, + "29": 2652.0, + "30": 2563.0, + "31": 2722.0, + "32": 2789.0, + "33": 3973.0, + "34": 2782.0, + "35": 527627.0, + "36": 527593.0, + "37": 25917.0, + "38": 527689.0, + "39": 527839.0, + "40": 527891.0, + "41": 31727.0, + "42": 527517.0, + "43": 528077.0, + "44": 527852.0, + "45": 30413.0, + "46": 527985.0, + "47": 528131.0, + "48": 528209.0, + "49": 528359.0, + "50": 550860.0, + "51": 560328.0, + "52": 533340.0, + "53": 1055795.0, + "54": 555563.0, + "55": 554263.0, + "56": 1053101.0, + "57": 1052681.0, + "58": 1079130.0, + "59": 557347.0, + "60": 582286.0, + "61": 587119.0, + "62": 1578262.0, + "63": 557738.0, + "64": 537139.0, + "65": 1578266.0, + "66": 1058838.0, + "67": 1056227.0, + "68": 1053806.0, + "69": 561478.0, + "70": 531123.0, + "71": 1053985.0, + "72": 1053905.0, + "73": 1075978.0, + "74": 1075340.0, + "75": 557909.0, + "76": 1578167.0, + "77": 552293.0, + "78": 556281.0, + "79": 1077972.0, + "80": 578039.0, + "81": 1058970.0, + "82": 1080698.0, + "83": 1084664.0, + "84": 575523.0, + "85": 560438.0, + "86": 554843.0, + "87": 575549.0, + "88": 536306.0, + "89": 1053569.0, + "90": 1578469.0, + "91": 1053432.0, + "92": 1053657.0, + "93": 529105.0, + "94": 1053708.0, + "95": 558441.0, + "96": 1055928.0, + "97": 1054101.0, + "98": 1054213.0, + "99": 554460.0, + "100": 1578715.0 } }, "mem-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1322251776.0, - "2": 1614032384.0, - "3": 1617393664.0, - "4": 1618344960.0, - "5": 1618880512.0, - "6": 1618880512.0, - "7": 1618880512.0, - "8": 1618880512.0, - "9": 1618880512.0, - "10": 1618880512.0, - "11": 1618880512.0, - "12": 1618880512.0, - "13": 1618880512.0, - "14": 1618880512.0, - "15": 1618880512.0, - "16": 1618880512.0, - "17": 1618880512.0, - "18": 1618880512.0, - "19": 1618880512.0, - "20": 1618880512.0, - "21": 1618880512.0, - "22": 1618880512.0, - "23": 1618880512.0, - "24": 1618880512.0, - "25": 1618880512.0, - "26": 1618880512.0, - "27": 1618880512.0, - "28": 1618880512.0, - "29": 1618880512.0, - "30": 1618880512.0, - "31": 1618880512.0, - "32": 1618880512.0, - "33": 1618880512.0, - "34": 1618880512.0, - "35": 1618880512.0, - "36": 1618880512.0, - "37": 1618880512.0, - "38": 1618880512.0, - "39": 1618880512.0, - "40": 1618880512.0, - "41": 1618880512.0, - "42": 1618880512.0, - "43": 1618880512.0, - "44": 1618880512.0, - "45": 1618880512.0, - "46": 1618880512.0, - "47": 1618880512.0, - "48": 1618880512.0, - "49": 1618880512.0, - "50": 1618880512.0, - "51": 1618880512.0, - "52": 1618880512.0, - "53": 1618880512.0, - "54": 1618880512.0, - "55": 1618880512.0, - "56": 1618880512.0, - "57": 1618880512.0, - "58": 1618880512.0, - "59": 1618880512.0, - "60": 1618880512.0, - "61": 1618880512.0, - "62": 1618880512.0, - "63": 1618880512.0, - "64": 1618880512.0, - "65": 1618880512.0, - "66": 1618880512.0, - "67": 1618880512.0, - "68": 1618880512.0, - "69": 1618880512.0, - "70": 1618880512.0, - "71": 1618880512.0, - "72": 1618880512.0, - "73": 1618880512.0, - "74": 1618880512.0, - "75": 1618880512.0, - "76": 1618880512.0, - "77": 1618880512.0, - "78": 1618880512.0, - "79": 1618880512.0, - "80": 1618880512.0, - "81": 1618880512.0, - "82": 1618880512.0, - "83": 1618880512.0, - "84": 1618880512.0, - "85": 1618880512.0, - "86": 1618880512.0, - "87": 1618880512.0, - "88": 1618880512.0, - "89": 1618880512.0, - "90": 1618880512.0, - "91": 1618880512.0, - "92": 1618880512.0, - "93": 1618880512.0, - "94": 1618880512.0, - "95": 1618880512.0, - "96": 1618880512.0, - "97": 1618880512.0, - "98": 1618880512.0, - "99": 1618880512.0, - "100": 1618880512.0 + "1": 1323515904.0, + "2": 1613952512.0, + "3": 1617445376.0, + "4": 1619704832.0, + "5": 1619704832.0, + "6": 1619704832.0, + "7": 1619704832.0, + "8": 1619704832.0, + "9": 1619704832.0, + "10": 1619704832.0, + "11": 1619704832.0, + "12": 1619704832.0, + "13": 1619704832.0, + "14": 1619704832.0, + "15": 1619704832.0, + "16": 1619704832.0, + "17": 1619704832.0, + "18": 1619704832.0, + "19": 1619704832.0, + "20": 1619704832.0, + "21": 1619704832.0, + "22": 1619704832.0, + "23": 1619704832.0, + "24": 1619704832.0, + "25": 1619704832.0, + "26": 1619704832.0, + "27": 1619704832.0, + "28": 1619704832.0, + "29": 1619704832.0, + "30": 1619704832.0, + "31": 1619704832.0, + "32": 1619704832.0, + "33": 1619704832.0, + "34": 1619704832.0, + "35": 1619704832.0, + "36": 1619704832.0, + "37": 1619704832.0, + "38": 1619704832.0, + "39": 1619704832.0, + "40": 1619704832.0, + "41": 1619704832.0, + "42": 1619704832.0, + "43": 1619704832.0, + "44": 1619704832.0, + "45": 1619704832.0, + "46": 1619704832.0, + "47": 1619704832.0, + "48": 1619704832.0, + "49": 1619704832.0, + "50": 1619704832.0, + "51": 1619704832.0, + "52": 1619704832.0, + "53": 1619704832.0, + "54": 1619704832.0, + "55": 1619704832.0, + "56": 1619704832.0, + "57": 1619704832.0, + "58": 1619704832.0, + "59": 1619704832.0, + "60": 1619704832.0, + "61": 1619704832.0, + "62": 1619704832.0, + "63": 1619704832.0, + "64": 1619704832.0, + "65": 1619704832.0, + "66": 1619704832.0, + "67": 1619704832.0, + "68": 1619704832.0, + "69": 1619704832.0, + "70": 1619704832.0, + "71": 1619704832.0, + "72": 1619704832.0, + "73": 1619704832.0, + "74": 1619704832.0, + "75": 1619704832.0, + "76": 1619704832.0, + "77": 1619704832.0, + "78": 1619704832.0, + "79": 1619704832.0, + "80": 1619704832.0, + "81": 1619704832.0, + "82": 1619704832.0, + "83": 1619704832.0, + "84": 1619704832.0, + "85": 1619704832.0, + "86": 1619704832.0, + "87": 1619704832.0, + "88": 1619704832.0, + "89": 1619704832.0, + "90": 1619704832.0, + "91": 1619704832.0, + "92": 1619704832.0, + "93": 1619704832.0, + "94": 1619704832.0, + "95": 1619704832.0, + "96": 1619704832.0, + "97": 1619704832.0, + "98": 1619704832.0, + "99": 1619704832.0, + "100": 1619704832.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.84271, - "3": 2.61896, - "4": 2.13682, - "5": 1.83117, - "6": 2.30687, - "7": 1.83781, - "8": 2.52546, - "9": 2.51587, - "10": 2.47099, - "11": 2.29751, - "12": 2.29851, - "13": 2.16383, - "14": 2.60155, - "15": 2.35852, - "16": 2.23005, - "17": 2.70719, - "18": 2.26641, - "19": 2.34687, - "20": 2.20341, - "21": 2.61366, - "22": 1.92204, - "23": 2.41045, - "24": 2.20076, - "25": 2.57863, - "26": 2.61356, - "27": 1.90975, - "28": 2.39018, - "29": 2.17417, - "30": 2.02774, - "31": 1.66802, - "32": 3.19679, - "33": 1.94532, - "34": 1.78092, - "35": 2.86854, - "36": 2.52329, - "37": 2.25087, - "38": 1.89933, - "39": 2.74747, - "40": 2.2918, - "41": 2.7229, - "42": 3.80161, - "43": 2.04029, - "44": 2.71436, - "45": 2.62618, - "46": 2.50912, - "47": 2.40089, - "48": 2.62069, - "49": 2.38881, - "50": 3.11118, - "51": 1.83181, - "52": 2.38462, - "53": 2.18831, - "54": 2.47774, - "55": 2.16497, - "56": 2.71397, - "57": 1.7184, - "58": 2.63468, - "59": 2.27191, - "60": 2.01431, - "61": 2.90864, - "62": 2.68672, - "63": 1.73088, - "64": 2.4613, - "65": 2.16857, - "66": 2.47629, - "67": 2.56543, - "68": 1.74081, - "69": 2.53246, - "70": 2.28935, - "71": 2.87404, - "72": 2.03657, - "73": 2.33536, - "74": 2.08138, - "75": 2.2141, - "76": 2.51275, - "77": 2.10365, - "78": 2.23134, - "79": 2.75405, - "80": 1.97207, - "81": 2.79415, - "82": 2.49662, - "83": 2.37386, - "84": 3.33686, - "85": 2.35332, - "86": 2.33909, - "87": 3.14205, - "88": 2.91235, - "89": 2.71384, - "90": 2.6279, - "91": 2.49967, - "92": 2.99414, - "93": 1.89073, - "94": 3.46814, - "95": 2.23133, - "96": 2.45399, - "97": 1.89708, - "98": 2.16621, - "99": 3.06519, - "100": 2.17963 + "2": 7.37367, + "3": 2.54182, + "4": 2.09823, + "5": 1.74432, + "6": 2.41953, + "7": 1.6559, + "8": 2.65326, + "9": 2.63405, + "10": 2.42359, + "11": 2.59476, + "12": 2.405, + "13": 2.23734, + "14": 2.72146, + "15": 2.33172, + "16": 2.2102, + "17": 2.95714, + "18": 2.20431, + "19": 2.49222, + "20": 2.29905, + "21": 2.71979, + "22": 2.18441, + "23": 2.55797, + "24": 2.18328, + "25": 2.83896, + "26": 2.91067, + "27": 2.21319, + "28": 2.38566, + "29": 2.12367, + "30": 1.73958, + "31": 1.76394, + "32": 2.99398, + "33": 1.96459, + "34": 1.68131, + "35": 2.60891, + "36": 2.28892, + "37": 2.15773, + "38": 1.92535, + "39": 2.59033, + "40": 2.07581, + "41": 2.90767, + "42": 2.62746, + "43": 1.98056, + "44": 2.76516, + "45": 2.50896, + "46": 2.01713, + "47": 2.14119, + "48": 2.18834, + "49": 2.0346, + "50": 2.00386, + "51": 1.6825, + "52": 2.76421, + "53": 2.50338, + "54": 2.64345, + "55": 2.225, + "56": 3.26343, + "57": 1.48644, + "58": 2.71536, + "59": 2.00729, + "60": 2.16353, + "61": 2.44532, + "62": 3.04555, + "63": 1.69774, + "64": 2.24699, + "65": 2.03727, + "66": 2.14857, + "67": 2.48781, + "68": 1.71264, + "69": 2.41294, + "70": 2.17268, + "71": 2.73027, + "72": 2.13735, + "73": 2.2041, + "74": 2.20307, + "75": 2.20197, + "76": 2.52796, + "77": 2.27171, + "78": 2.21398, + "79": 2.90986, + "80": 2.33253, + "81": 2.7207, + "82": 2.52303, + "83": 2.27444, + "84": 1.94576, + "85": 2.07223, + "86": 2.12538, + "87": 2.79202, + "88": 2.4337, + "89": 2.17604, + "90": 2.44102, + "91": 2.46034, + "92": 2.84467, + "93": 1.98126, + "94": 2.94718, + "95": 2.3739, + "96": 2.44264, + "97": 2.2023, + "98": 2.19249, + "99": 3.07219, + "100": 2.08776 } } } \ No newline at end of file