all_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP5yHiaAgOQY1wAgXj5PEkb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DiveInKU/jaso-me-ai/blob/main/all_processing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "68eS1iSQFScX",
        "outputId": "8e65e469-a485-4beb-bec6-9627c0b08bd0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "/content/drive/MyDrive/Colab Notebooks/jasome-fastapi\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd '/content/drive/MyDrive/Colab Notebooks/jasome-fastapi'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import re"
      ],
      "metadata": {
        "id": "7b7_gf6PGo6n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def process(data, file_name):\n",
        "  result = ''\n",
        "  train = ''\n",
        "  print(len(data))\n",
        "  print(int(len(data)*0.9))\n",
        "  try:\n",
        "    for i, (company, field, spec, question, content) in enumerate(data[1:]):\n",
        "      if re.match(r\".*\\(.*\\)\",company):\n",
        "        # 회사명에 (주), (학) 등 제거\n",
        "        company = re.sub(r\"\\(.*\\)\",\"\", company)\n",
        "      company = company.replace(\"㈜\", \"\")\n",
        "      # 시작과 끝에 [\" \"] 제거\n",
        "      content = re.sub(r\"^\\[[\\'\\\"]\",\"\", content)\n",
        "      content = re.sub(r\"[\\'\\\"]\\]$\",\"\", content) \n",
        "      # 글자수 ~~ byte 제거 \n",
        "      content = re.sub(r\"\\\\n글자수.*Byte\", \"\", content) \n",
        "      # 소제목은 대부분 처음에 \"로 바로 시작하고 줄바꿈 두 번 하는 특성이 있음\n",
        "      content = re.sub(r\"\\s*\\\".*?\\\"\", \"\", content)\n",
        "      content = re.sub(r\"\\s*\\'.*?\\'\", \"\", content)\n",
        "      content = re.sub(r\"\\s*\\[.*?\\]\", \"\", content)\n",
        "      # 두번 줄바꿈 한번으로\n",
        "      content = re.sub(r\"\\\\n\\\\n\", \"\\n\", content) \n",
        "      # \\n을 진짜 개행문자로 변환\n",
        "      content = content.replace(r\"\\n\", \"\\n\")\n",
        "      # . 를 .\\n으로 변환\n",
        "      content = re.sub(r\"(?<=\\.) \", \"\\n\", content)\n",
        "      content = re.sub(r\"(?<=\\.)\\n\", \"\\n\", content)\n",
        "\n",
        "      # 잡코리아에서 추가된 문자열 제거\n",
        "      # 아쉬운점 제거\n",
        "      content = re.sub(r\"아쉬운점 \\d\", \"\", content)\n",
        "      # 좋은점 제거\n",
        "      content = re.sub(r\"좋은점 \\d\", \"\", content)\n",
        "\n",
        "      # 숫자와 특수문자만 있다면 제거\n",
        "      content = re.sub(r\"^[\\{\\}\\[\\]\\/?\\.,;:|\\)*~`!^\\-_+<>@\\#$%&\\\\\\=\\(\\'\\\"\\d\\s]*$\", \"\", content)\n",
        "      # 넘버링 제거\n",
        "      content = re.sub(r\"\\d[\\{\\}\\[\\]\\/?\\.,;:|\\)*~`!^\\-_+<>@\\#$%&\\\\\\=\\(\\'\\\"]+\", \"\", content)\n",
        "      # 공백 제거\n",
        "      content = content.strip()\n",
        "      # 불규칙적인 특수문자 제거\n",
        "      content = content.replace('[', \"\")\n",
        "      content = content.replace(']', \"\")\n",
        "      content = content.replace('\\\\', \"\")\n",
        "      content = content.replace('<', \"\")\n",
        "      content = content.replace('>', \"\")\n",
        "      content = content.replace('≪', \"\")\n",
        "      content = content.replace('≫', \"\")\n",
        "      content = content.replace('//', '')\n",
        "\n",
        "      # 공백만 있거나 빈 문자열일 경우 제거\n",
        "      if content==\"\" or content.isspace():\n",
        "        continue\n",
        "      # 회사 이름 제거\n",
        "      content = content.replace(company, \"<귀사>\")\n",
        "\n",
        "      # 영어 자소서인 경우 건너뜀\n",
        "      if not re.match(r\"[ㄱ-ㅣ가-힣]+\", content):\n",
        "        continue\n",
        "\n",
        "      if i < int(len(data)*0.9):\n",
        "        result += content + '\\n'\n",
        "      else:\n",
        "        train += content + '\\n'\n",
        "  except ValueError as e:\n",
        "    pass\n",
        "  # output = open(\"../after/train/{}_train.txt\".format(file_name), 'w')\n",
        "  # output.write(result)\n",
        "  # output.close()\n",
        "  # output2 = open(\"../after/test/{}_test.txt\".format(file_name), 'w')\n",
        "  # output2.write(train)\n",
        "  # output2.close()\n",
        "  all_output = open(\"../merged/all_train.txt\", 'w')\n",
        "  all_output.write(result)\n",
        "  all_output.close()\n",
        "  all_output2 = open(\"../merged/all_test.txt\", 'w')\n",
        "  all_output2.write(train)\n",
        "  all_output2.close()"
      ],
      "metadata": {
        "id": "AMru2qIhIcip"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "metadata": {
        "id": "22o5j_Gdyple"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wc_k0XXUy1mh",
        "outputId": "d88804dd-b8cd-4f93-cb1f-3183286b9dfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/Colab Notebooks/jasome-fastapi/before\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(os.getcwd())\n",
        "# os.chdir('./before')\n",
        "print(os.listdir())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rStDrIK8yrDU",
        "outputId": "edad926b-3b1a-4ee4-e786-88f9a2c7e288"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/Colab Notebooks/jasome-fastapi/before\n",
            "['selfs_sales_meta.csv', 'selfs_businees_meta.csv', 'selfs_marketing_meta.csv', 'selfs_produce_meta.csv', 'selfs_it_meta.csv']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for original_file_name in os.listdir():\n",
        "  f = open(original_file_name, 'r')\n",
        "  rdr = csv.reader(f)\n",
        "  data = list(rdr)\n",
        "  f.close()\n",
        "  print(os.path.splitext(original_file_name)[0])\n",
        "  process(data, os.path.splitext(original_file_name)[0])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VajmtaQNznUe",
        "outputId": "807af225-17f6-40e7-ab93-39c9a9207462"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "selfs_sales_meta\n",
            "1218\n",
            "1096\n",
            "selfs_businees_meta\n",
            "1557\n",
            "1401\n",
            "selfs_marketing_meta\n",
            "881\n",
            "792\n",
            "selfs_produce_meta\n",
            "1117\n",
            "1005\n",
            "selfs_it_meta\n",
            "1010\n",
            "909\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jpype1\n",
        "!pip install konlpy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EQ6QuM0uqet3",
        "outputId": "9a89671e-45d9-4767-d061-34b6f6c9bb62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: jpype1 in /usr/local/lib/python3.7/dist-packages (1.4.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from jpype1) (4.1.1)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: konlpy in /usr/local/lib/python3.7/dist-packages (0.6.0)\n",
            "Requirement already satisfied: JPype1>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.4.0)\n",
            "Requirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.21.6)\n",
            "Requirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (4.9.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from JPype1>=0.7.0->konlpy) (4.1.1)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from konlpy.tag import Kkma\n",
        "kkma = Kkma()"
      ],
      "metadata": {
        "id": "yttMQeA4sNn9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "company_names = []\n",
        "for word, pos in kkma.pos('우리에프아이에스'):\n",
        "  print(word, pos)\n",
        "  if pos == 'NNP':\n",
        "    company_names.append(word)\n",
        "print(company_names)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JzBmnhrmsUiC",
        "outputId": "d561713d-4655-445c-a214-40e2179f492e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "우리 NP\n",
            "에프 NNG\n",
            "아이 NNG\n",
            "에 JKM\n",
            "스 VV\n",
            "[]\n"
          ]
        }
      ]
    }
  ]
}