SimTCP/grade.py at master · SongClass/SimTCP · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#!/usr/bin/env python3
import os
import signal
from subprocess import Popen, PIPE

import subprocess
import enum
import collections
import time
import sys


class TestResultType(enum.Enum):
    TRANSFER_ERROR = 0
    TRANSFER_TIMEOUT = 1
    SLOW_TRANSFER = 2
    FAST_TRANSFER = 3

TestArguments = collections.namedtuple('TestArguments', "file loss delay buffer")
TestCase = collections.namedtuple("TestCase", "args secs")
TestResult = collections.namedtuple("TestResult", "type duration")

def run_test_case(case: TestCase) -> TestResult:
    full_credit_time = case.secs * 1.5
    half_credit_time = case.secs * 2.0
    start_time = time.time()

    test_args = ["python3", "tester.py", "--file", case.args.file, "--loss",
                 case.args.loss, "--buffer", case.args.buffer,
                 "--delay", case.args.delay]

    try:
        subprocess.run(test_args, timeout=half_credit_time, check=True,
                       stdout=subprocess.PIPE)
    except subprocess.TimeoutExpired:
        return TestResult(TestResultType.TRANSFER_TIMEOUT, None)
    except subprocess.CalledProcessError:
        return TestResult(TestResultType.TRANSFER_ERROR, None)

    end_time = time.time()
    duration = end_time - start_time
    if duration <= full_credit_time:
        result_type = TestResultType.FAST_TRANSFER
    else:
        result_type = TestResultType.SLOW_TRANSFER
    return TestResult(result_type, duration)


TEST_CASES = [
    TestCase(TestArguments("grading_data/test-1.bin", "0.15", "0.05", "10"), 9),
    TestCase(TestArguments("grading_data/test-1.bin", "0.5", "0.05", "10"), 15),
    TestCase(TestArguments("grading_data/test-2.bin", "0.15", "0.01", "20"), 12),
    TestCase(TestArguments("grading_data/test-3.bin", "0.01", "0.01", "50"), 16),
    TestCase(TestArguments("grading_data/test-3.bin", "0.05", "0.01", "50"), 40),
    TestCase(TestArguments("grading_data/test-4.bin", "0.02", "0.01", "50"), 50),
    TestCase(TestArguments("grading_data/test-4.bin", "0.05", "0.02", "50"), 50),
    TestCase(TestArguments("grading_data/test-5.bin", "0.05", "0.01", "50"), 70),
    TestCase(TestArguments("grading_data/test-6.bin", "0.02", "0.01", "50"), 80),
    TestCase(TestArguments("grading_data/test-6.bin", "0.03", "0.01", "50"), 80),

    # TestCase(TestArguments("grading_data/test-1.bin", "0", ".5", "10"), 9),
    # TestCase(TestArguments("grading_data/test-1.bin", "0", ".5", "10"), 15),
    # TestCase(TestArguments("grading_data/test-2.bin", "0", "0.1", "20"), 12),
    # TestCase(TestArguments("grading_data/test-3.bin", "0", ".01", "30"), 16),
    # TestCase(TestArguments("grading_data/test-3.bin", "0", ".01", "30"), 25),
    # TestCase(TestArguments("grading_data/test-4.bin", "0", ".1", "40"), 9),
    # TestCase(TestArguments("grading_data/test-4.bin", "0", ".2", "40"), 15),
    # TestCase(TestArguments("grading_data/test-5.bin", "0", ".01", "500"), 45),
    # TestCase(TestArguments("grading_data/test-6.bin", "0", ".02", "600"), 52),
    # TestCase(TestArguments("grading_data/test-6.bin", "0", ".01", "300"), 60),


    # TestCase(TestArguments("grading_data/test-1.bin", "0", ".5", "10"), 7),
    # TestCase(TestArguments("grading_data/test-2.bin", "0", "0.1", "20"), 5),
    # TestCase(TestArguments("grading_data/test-3.bin", "0", ".01", "30"), 6),
    # TestCase(TestArguments("grading_data/test-4.bin", "0.1", ".1", "40"), 6),
    # TestCase(TestArguments("grading_data/test-5.bin", "0.1", ".01", "50"), 32),
    # TestCase(TestArguments("grading_data/test-6.bin", "0", ".01", "60"), 12),
]

TOTAL_SCORE = 0
SUC_CASE = 0
MAX_SCORE = 20
MAX_CASE = 10
TOTAL_TIME = 0
TIME_LIST = []
for CASE_NUM, A_CASE in enumerate(TEST_CASES):
    FULL_CREDIT_TIME = A_CASE.secs * 1.5
    HALF_CREDIT_TIME = FULL_CREDIT_TIME * 1.5
    print("Test case: {}".format(CASE_NUM + 1))
    print("  - File: {}".format(A_CASE.args.file))
    print("  - Loss: {}%".format(100 * float(A_CASE.args.loss)))
    print("  - Delay: {} seconds".format(A_CASE.args.delay))
    print("  - Buffer: {} packets".format(A_CASE.args.buffer))
    print("  - Full credit: {} secs".format(round(FULL_CREDIT_TIME)))
    print("  - Half credit: {} secs".format(round(HALF_CREDIT_TIME)))
    print("----------")

    port = 9999
    process = Popen(["lsof", "-i", ":{0}".format(port)], stdout=PIPE, stderr=PIPE)
    stdout, stderr = process.communicate()
    for process in str(stdout.decode("utf-8")).split("\n")[1:]:
        data = [x for x in process.split(" ") if x != '']
        if (len(data) <= 1):
            continue

        os.kill(int(data[1]), signal.SIGKILL)


    CASE_RESULT = run_test_case(A_CASE)
    TIME_LIST.append(CASE_RESULT.duration)
    if CASE_RESULT.duration:
        TOTAL_TIME += CASE_RESULT.duration
    if CASE_RESULT.type == TestResultType.TRANSFER_ERROR:
        print("  ! 0 points, file was transferred incorrectly")
    elif CASE_RESULT.type == TestResultType.TRANSFER_TIMEOUT:
        print("  ! 0 points, file not transferred in {} secs".format(
            round(HALF_CREDIT_TIME)))
    elif CASE_RESULT.type == TestResultType.SLOW_TRANSFER:
        print("  ~ 1 point, transferred in {} secs".format(round(CASE_RESULT.duration)))
        TOTAL_SCORE += 1
        SUC_CASE += 1
    elif CASE_RESULT.type == TestResultType.FAST_TRANSFER:
        print("  * 2 points, transferred in {} secs".format(round(CASE_RESULT.duration)))
        TOTAL_SCORE += 1
        SUC_CASE += 1
    print("")

print("==========")
print("Success Case: {} / {}".format(SUC_CASE, MAX_CASE))

print("Time: {} ".format(TIME_LIST))
print("Total Time: {} ".format(TOTAL_TIME))
sys.exit(0 if TOTAL_SCORE == MAX_SCORE else 1)