Domovská stránka Prehľadávať Pomoc
Registrovať Prihlásiť sa
github
/
GLM-130B
zrkadlo https://github.com/THUDM/GLM-130B.git
2
0
Fork 0
Súbory Issues 0 Wiki
Strom: 737be7c740
Branche Tagy
GLM-130B
/
tasks
/
mmlu
/
task.py

task.py 2.0 KB
História Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778 
  1. import numpy as np
    2. 

  2. 

  3. from typing import Dict, Tuple
    4. 

  4. 

  5. from evaluation import MultiChoiceTask
    6. 

  6. 

  7. categories = {
    8. 

  8.     "STEM": [
    9. 

  9.         "Abstract Algebra",
    10. 

  10.         "Anatomy",
    11. 

  11.         "Astronomy",
    12. 

  12.         "College Biology",
    13. 

  13.         "College Chemistry",
    14. 

  14.         "College Computer Science",
    15. 

  15.         "College Mathematics",
    16. 

  16.         "College Physics",
    17. 

  17.         "Computer Security",
    18. 

  18.         "Conceptual Physics",
    19. 

  19.         "Electrical Engineering",
    20. 

  20.         "Elementary Mathematics",
    21. 

  21.         "High School Biology",
    22. 

  22.         "High School Chemistry",
    23. 

  23.         "High School Computer Science",
    24. 

  24.         "High School Mathematics",
    25. 

  25.         "High School Physics",
    26. 

  26.         "High School Statistics",
    27. 

  27.         "Machine Learning",
    28. 

  28.     ],
    29. 

  29.     "Other": [
    30. 

  30.         "Business Ethics",
    31. 

  31.         "Clinical Knowledge",
    32. 

  32.         "College Medicine",
    33. 

  33.         "Global Facts",
    34. 

  34.         "Human Aging",
    35. 

  35.         "Management",
    36. 

  36.         "Marketing",
    37. 

  37.         "Medical Genetics",
    38. 

  38.         "Miscellaneous",
    39. 

  39.         "Nutrition",
    40. 

  40.         "Professional Accounting",
    41. 

  41.         "Professional Medicine",
    42. 

  42.         "Virology",
    43. 

  43.     ],
    44. 

  44.     "Social Sciences": [
    45. 

  45.         "Econometrics",
    46. 

  46.         "High School Geography",
    47. 

  47.         "High School Government and Politics",
    48. 

  48.         "High School Macroeconomics",
    49. 

  49.         "High School Microeconomics",
    50. 

  50.         "High School Psychology",
    51. 

  51.         "Human Sexuality",
    52. 

  52.         "Professional Psychology",
    53. 

  53.         "Public Relations",
    54. 

  54.         "Security Studies",
    55. 

  55.         "Sociology",
    56. 

  56.         "US Foreign Policy",
    57. 

  57.     ],
    58. 

  58.     "Humanities": [
    59. 

  59.         "Formal Logic",
    60. 

  60.         "High School European History",
    61. 

  61.         "High School US History",
    62. 

  62.         "High School World History",
    63. 

  63.         "International Law",
    64. 

  64.         "Jurisprudence",
    65. 

  65.         "Logical Fallacies",
    66. 

  66.         "Moral Disputes",
    67. 

  67.         "Moral Scenarios",
    68. 

  68.         "Philosophy",
    69. 

  69.         "Prehistory",
    70. 

  70.         "Professional Law",
    71. 

  71.         "World Religions",
    72. 

  72.     ],
    73. 

  73. }
    74. 

  74. 

  75. 

  76. class MMLU(MultiChoiceTask):
    77. 

  77.     def report_overall_metrics(self, result_dict_all: Dict[str, Tuple[Dict[str, float], int]]):
    78. 

  78.         self.report_group_metrics("Overall", result_dict_all, level=0)
    79.