We Keep Coding

How to append dataframes to the same excel sheet in pandas [duplicate]

I currently have this code. It works perfectly.
It loops through excel files in a folder,
removes the first 2 rows, then saves them as individual excel files,
and it also saves the files in the loop as an appended file.
Currently the appended file overwrites the existing file each time I run the code.
I need to append the new data to the bottom of the already existing excel sheet ('master_data.xlsx)
dfList = []
path = 'C:\\Test\\TestRawFile'
newpath = 'C:\\Path\\To\\New\\Folder'
for fn in os.listdir(path):
# Absolute file path
file = os.path.join(path, fn)
if os.path.isfile(file):
# Import the excel file and call it xlsx_file
xlsx_file = pd.ExcelFile(file)
# View the excel files sheet names
xlsx_file.sheet_names
# Load the xlsx files Data sheet as a dataframe
df = xlsx_file.parse('Sheet1',header= None)
df_NoHeader = df[2:]
data = df_NoHeader
# Save individual dataframe
data.to_excel(os.path.join(newpath, fn))
dfList.append(data)
appended_data = pd.concat(dfList)
appended_data.to_excel(os.path.join(newpath, 'master_data.xlsx'))
I thought this would be a simple task, but I guess not.
I think I need to bring in the master_data.xlsx file as a dataframe, then match the index up with the new appended data, and save it back out. Or maybe there is an easier way. Any Help is appreciated.

UPDATE [2022-01-08]: starting from version 1.4.0 Pandas supports appending to existing Excel sheet, preserving the old contents, "out of the box"!
Good job Pandas Team!
Excerpt from the ExcelWriter documentation:
if_sheet_exists : {'error', 'new', 'replace', 'overlay'}, default 'error'
How to behave when trying to write to a sheet that already
exists (append mode only).
...
* overlay: Write contents to the existing sheet without removing the old contents.
.. versionadded:: 1.3.0
.. versionchanged:: 1.4.0
Added ``overlay`` option
For Pandas versions < 1.4.0 please find below a helper function for appending a Pandas DataFrame to an existing Excel file.
If an Excel file doesn't exist then it will be created.
UPDATE [2021-09-12]: fixed for Pandas 1.3.0+
The following functions have been tested with:
Pandas 1.3.2
OpenPyxl 3.0.7
from pathlib import Path
from copy import copy
from typing import Union, Optional
import numpy as np
import pandas as pd
import openpyxl
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
def copy_excel_cell_range(
src_ws: openpyxl.worksheet.worksheet.Worksheet,
min_row: int = None,
max_row: int = None,
min_col: int = None,
max_col: int = None,
tgt_ws: openpyxl.worksheet.worksheet.Worksheet = None,
tgt_min_row: int = 1,
tgt_min_col: int = 1,
with_style: bool = True
) -> openpyxl.worksheet.worksheet.Worksheet:
"""
copies all cells from the source worksheet [src_ws] starting from [min_row] row
and [min_col] column up to [max_row] row and [max_col] column
to target worksheet [tgt_ws] starting from [tgt_min_row] row
and [tgt_min_col] column.
#param src_ws: source worksheet
#param min_row: smallest row index in the source worksheet (1-based index)
#param max_row: largest row index in the source worksheet (1-based index)
#param min_col: smallest column index in the source worksheet (1-based index)
#param max_col: largest column index in the source worksheet (1-based index)
#param tgt_ws: target worksheet.
If None, then the copy will be done to the same (source) worksheet.
#param tgt_min_row: target row index (1-based index)
#param tgt_min_col: target column index (1-based index)
#param with_style: whether to copy cell style. Default: True
#return: target worksheet object
"""
if tgt_ws is None:
tgt_ws = src_ws
# https://stackoverflow.com/a/34838233/5741205
for row in src_ws.iter_rows(min_row=min_row, max_row=max_row,
min_col=min_col, max_col=max_col):
for cell in row:
tgt_cell = tgt_ws.cell(
row=cell.row + tgt_min_row - 1,
column=cell.col_idx + tgt_min_col - 1,
value=cell.value
)
if with_style and cell.has_style:
# tgt_cell._style = copy(cell._style)
tgt_cell.font = copy(cell.font)
tgt_cell.border = copy(cell.border)
tgt_cell.fill = copy(cell.fill)
tgt_cell.number_format = copy(cell.number_format)
tgt_cell.protection = copy(cell.protection)
tgt_cell.alignment = copy(cell.alignment)
return tgt_ws
def append_df_to_excel(
filename: Union[str, Path],
df: pd.DataFrame,
sheet_name: str = 'Sheet1',
startrow: Optional[int] = None,
max_col_width: int = 30,
autofilter: bool = False,
fmt_int: str = "#,##0",
fmt_float: str = "#,##0.00",
fmt_date: str = "yyyy-mm-dd",
fmt_datetime: str = "yyyy-mm-dd hh:mm",
truncate_sheet: bool = False,
storage_options: Optional[dict] = None,
**to_excel_kwargs
) -> None:
"""
Append a DataFrame [df] to existing Excel file [filename]
into [sheet_name] Sheet.
If [filename] doesn't exist, then this function will create it.
#param filename: File path or existing ExcelWriter
(Example: '/path/to/file.xlsx')
#param df: DataFrame to save to workbook
#param sheet_name: Name of sheet which will contain DataFrame.
(default: 'Sheet1')
#param startrow: upper left cell row to dump data frame.
Per default (startrow=None) calculate the last row
in the existing DF and write to the next row...
#param max_col_width: maximum column width in Excel. Default: 40
#param autofilter: boolean - whether add Excel autofilter or not. Default: False
#param fmt_int: Excel format for integer numbers
#param fmt_float: Excel format for float numbers
#param fmt_date: Excel format for dates
#param fmt_datetime: Excel format for datetime's
#param truncate_sheet: truncate (remove and recreate) [sheet_name]
before writing DataFrame to Excel file
#param storage_options: dict, optional
Extra options that make sense for a particular storage connection, e.g. host, port,
username, password, etc., if using a URL that will be parsed by fsspec, e.g.,
starting “s3://”, “gcs://”.
#param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()`
[can be a dictionary]
#return: None
Usage examples:
>>> append_df_to_excel('/tmp/test.xlsx', df, autofilter=True,
freeze_panes=(1,0))
>>> append_df_to_excel('/tmp/test.xlsx', df, header=None, index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False, startrow=25)
>>> append_df_to_excel('/tmp/test.xlsx', df, index=False,
fmt_datetime="dd.mm.yyyy hh:mm")
(c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile)
"""
def set_column_format(ws, column_letter, fmt):
for cell in ws[column_letter]:
cell.number_format = fmt
filename = Path(filename)
file_exists = filename.is_file()
# process parameters
# calculate first column number
# if the DF will be written using `index=True`, then `first_col = 2`, else `first_col = 1`
first_col = int(to_excel_kwargs.get("index", True)) + 1
# ignore [engine] parameter if it was passed
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
# save content of existing sheets
if file_exists:
wb = load_workbook(filename)
sheet_names = wb.sheetnames
sheet_exists = sheet_name in sheet_names
sheets = {ws.title: ws for ws in wb.worksheets}
with pd.ExcelWriter(
filename.with_suffix(".xlsx"),
engine="openpyxl",
mode="a" if file_exists else "w",
if_sheet_exists="new" if file_exists else None,
date_format=fmt_date,
datetime_format=fmt_datetime,
storage_options=storage_options
) as writer:
if file_exists:
# try to open an existing workbook
writer.book = wb
# get the last row in the existing Excel sheet
# if it was not specified explicitly
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
# truncate sheet
if truncate_sheet and sheet_name in writer.book.sheetnames:
# index of [sheet_name] sheet
idx = writer.book.sheetnames.index(sheet_name)
# remove [sheet_name]
writer.book.remove(writer.book.worksheets[idx])
# create an empty sheet [sheet_name] using old index
writer.book.create_sheet(sheet_name, idx)
# copy existing sheets
writer.sheets = sheets
else:
# file doesn't exist, we are creating a new one
startrow = 0
# write out the DataFrame to an ExcelWriter
df.to_excel(writer, sheet_name=sheet_name, **to_excel_kwargs)
worksheet = writer.sheets[sheet_name]
if autofilter:
worksheet.auto_filter.ref = worksheet.dimensions
for xl_col_no, dtyp in enumerate(df.dtypes, first_col):
col_no = xl_col_no - first_col
width = max(df.iloc[:, col_no].astype(str).str.len().max(),
len(df.columns[col_no]) + 6)
width = min(max_col_width, width)
column_letter = get_column_letter(xl_col_no)
worksheet.column_dimensions[column_letter].width = width
if np.issubdtype(dtyp, np.integer):
set_column_format(worksheet, column_letter, fmt_int)
if np.issubdtype(dtyp, np.floating):
set_column_format(worksheet, column_letter, fmt_float)
if file_exists and sheet_exists:
# move (append) rows from new worksheet to the `sheet_name` worksheet
wb = load_workbook(filename)
# retrieve generated worksheet name
new_sheet_name = set(wb.sheetnames) - set(sheet_names)
if new_sheet_name:
new_sheet_name = list(new_sheet_name)[0]
# copy rows written by `df.to_excel(...)` to
copy_excel_cell_range(
src_ws=wb[new_sheet_name],
tgt_ws=wb[sheet_name],
tgt_min_row=startrow + 1,
with_style=True
)
# remove new (generated by Pandas) worksheet
del wb[new_sheet_name]
wb.save(filename)
wb.close()
Old version (tested with Pandas 1.2.3 and Openpyxl 3.0.5):
import os
from openpyxl import load_workbook
def append_df_to_excel(filename, df, sheet_name='Sheet1', startrow=None,
truncate_sheet=False,
**to_excel_kwargs):
"""
Append a DataFrame [df] to existing Excel file [filename]
into [sheet_name] Sheet.
If [filename] doesn't exist, then this function will create it.
#param filename: File path or existing ExcelWriter
(Example: '/path/to/file.xlsx')
#param df: DataFrame to save to workbook
#param sheet_name: Name of sheet which will contain DataFrame.
(default: 'Sheet1')
#param startrow: upper left cell row to dump data frame.
Per default (startrow=None) calculate the last row
in the existing DF and write to the next row...
#param truncate_sheet: truncate (remove and recreate) [sheet_name]
before writing DataFrame to Excel file
#param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()`
[can be a dictionary]
#return: None
Usage examples:
>>> append_df_to_excel('d:/temp/test.xlsx', df)
>>> append_df_to_excel('d:/temp/test.xlsx', df, header=None, index=False)
>>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2',
index=False)
>>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2',
index=False, startrow=25)
(c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile)
"""
# Excel file doesn't exist - saving and exiting
if not os.path.isfile(filename):
df.to_excel(
filename,
sheet_name=sheet_name,
startrow=startrow if startrow is not None else 0,
**to_excel_kwargs)
return
# ignore [engine] parameter if it was passed
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
writer = pd.ExcelWriter(filename, engine='openpyxl', mode='a')
# try to open an existing workbook
writer.book = load_workbook(filename)
# get the last row in the existing Excel sheet
# if it was not specified explicitly
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
# truncate sheet
if truncate_sheet and sheet_name in writer.book.sheetnames:
# index of [sheet_name] sheet
idx = writer.book.sheetnames.index(sheet_name)
# remove [sheet_name]
writer.book.remove(writer.book.worksheets[idx])
# create an empty sheet [sheet_name] using old index
writer.book.create_sheet(sheet_name, idx)
# copy existing sheets
writer.sheets = {ws.title:ws for ws in writer.book.worksheets}
if startrow is None:
startrow = 0
# write out the new sheet
df.to_excel(writer, sheet_name, startrow=startrow, **to_excel_kwargs)
# save the workbook
writer.save()
Usage examples:
filename = r'C:\OCC.xlsx'
append_df_to_excel(filename, df)
append_df_to_excel(filename, df, header=None, index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False, startrow=25)
c:/temp/test.xlsx:
PS you may also want to specify header=None if you don't want to duplicate column names...
UPDATE: you may also want to check this old solution

If you aren't strictly looking for an excel file, then get the output as csv file and just copy the csv to a new excel file.
Note: this only works when you have less than 1000 columns since csv has a limit on the number of columns you can write.
df.to_csv('filepath', mode='a', index = False, header=None)
mode='a' means append.
This is a roundabout way it but works neat!

Building on MaxU and others' code and comments but simplifying to only fix the bug with pandas ExcelWriter that causes to_excel to create a new sheet rather than append to an existing sheet in append mode.
As others have noted, to_excel uses the ExcelWriter.sheets property and this is not populated when by ExcelWriter.
Fix is a one liner, otherwise code is standard pandas approach as documented in to_excel.
# xl_path is destination xlsx spreadsheet
with pd.ExcelWriter(xl_path, 'openpyxl', mode='a') as writer:
# fix line
writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets)
df.to_excel(writer, sheet_name)

import pandas as pd
import openpyxl
workbook = openpyxl.load_workbook("test.xlsx")
writer = pd.ExcelWriter('test.xlsx', engine='openpyxl')
writer.book = workbook
writer.sheets = dict((ws.title, ws) for ws in workbook.worksheets)
data_df.to_excel(writer, 'Existing_sheetname')
writer.save()
writer.close()

If you use ExcelWriter on the sheet every time it is going to override the previous sheet and all that will be visible is the last data sheet you appended to the workbook.
Instead you can maintain a counter that is 1 initially for which you need to initialize the excel sheet and add initial data using the existing approach of
writer = pd.ExcelWriter(output_file, engine='openpyxl')
df = pd.read_excel(output_file, sheet_name='TestSheet1')
or you can use the following approach i used. to load the workbook next time you want to use it or else file not find exception if you try to load it in the first case.
USage:
from bs4 import BeautifulSoup
import requests
import pandas as pd
from openpyxl import load_workbook
urls = ["http://millenniumcricketleague.com/Home/ShowTeam.aspx?tid=22",
"http://millenniumcricketleague.com/Home/ShowTeam.aspx?tid=40"]
path = "F:\meta_1.xlsx"
writer = pd.ExcelWriter(path,engine='openpyxl')
counter = 1
for url in urls:
table_data = []
final = []
html_content = requests.get(url).text
soup = BeautifulSoup(html_content, "lxml")
x = soup.find_all('table')
for table in x[1:]:
for tr in table.find_all("tr"):
newrow = []
for td in tr.find_all("td"):
newrow.append(td.text.replace('\n', ' ').strip())
table_data.append(newrow)
df = pd.DataFrame(table_data)
sheetname = 'Sheet%s' % counter
if(counter!=1):
writer.book = load_workbook(path)
df.to_excel(writer, sheet_name=sheetname)
counter = counter + 1
writer.save()
NO need to close the excelwriter. its an automatic function. Will show you a warning if you define it explicitly

This worked for me
import os
import openpyxl
import pandas as pd
from openpyxl.utils.dataframe import dataframe_to_rows
file = r"myfile.xlsx"
df = pd.DataFrame({'A': 1, 'B': 2})
# create excel file
if os.path.isfile(file): # if file already exists append to existing file
workbook = openpyxl.load_workbook(file) # load workbook if already exists
sheet = workbook['my_sheet_name'] # declare the active sheet
# append the dataframe results to the current excel file
for row in dataframe_to_rows(df, header = False, index = False):
sheet.append(row)
workbook.save(file) # save workbook
workbook.close() # close workbook
else: # create the excel file if doesn't already exist
with pd.ExcelWriter(path = file, engine = 'openpyxl') as writer:
df.to_excel(writer, index = False, sheet_name = 'my_sheet_name')

This question has been out here a while. The answer is ok, but I believe this will solve most peoples question.
simply use glob to access the files in a specific directory, loop through them, create a dataframe of each file, append it to the last one, then export to a folder. I also included commented out code to run through this with csvs.
import os
import pandas as pd
import glob
# put in path to folder with files you want to append
# *.xlsx or *.csv will get all files of that type
path = "C:/Users/Name/Folder/*.xlsx"
#path = "C:/Users/Name/Folder/*.csv"
# initialize a empty df
appended_data = pd.DataFrame()
#loop through each file in the path
for file in glob.glob(path):
print(file)
# create a df of that file path
df = pd.read_excel(file, sheet_name = 0)
#df = pd.read_csv(file, sep=',')
# appened it
appended_data = appended_data.append(df)
appended_data
# export the appeneded data to a folder of your choice
exportPath = 'C:/My/EXPORT/PATH/appended_dataExport.csv'
appended_data.to_csv(os.path.join(exportPath),index=False)

Complementing to #david, if you dont care the index and you can use .csv, this function helps to append any df to an existing csv
def append_df(self, path_file, df):
with open(path_file, 'a+') as f:
df.to_csv(f, header=f.tell() == 0, encoding='utf-8', index=False)
Notes:
a+ create the file if it doesnot exist
f.tell() == 0 add header if the first row

from openpyxl import load_workbook
wb = load_workbook(filepath)
ws = wb["Sheet1"]
df = dataframe.values.tolist()
for i in range(len(df)):
ws.append(df[i])
wb.save(filepath)

Append DataFrame to existing excel file
Use ExcelWriter to append DataFrame to an existing excel file. This is a simple approach and uses the existing library features.
with pd.ExcelWriter('existing_excel_file.xlsx',mode='a') as writer:
df.to_excel(writer, sheet_name='existing_sheet_name')
For detailed examples refer to pandas read Excel File with Examples

Pandas creates new excel sheet when trying to append to existing sheet

I have the code where I want to read data from the current sheet, store it in df_old, append the current data to it using df = df_old.append(df) and then replace the data in the sheet with this new dataframe. However, what it does instead is create a new sheet with the exact same name where it publishes this new dataframe. I tried adding if_sheet_exists="replace" as an argument to ExcelWriter but this did not change anything. How can I force it to overwrite the data in the sheet with the current name?
df_old = pd.read_excel(r'C:\Users\XXX\Downloads\Digitalisation\mat_flow\reblend_v2.xlsx',sheet_name = ft_tags_final[i][j])
df = df_old.append(df)
with pd.ExcelWriter(r'C:\Users\XXX\Downloads\Digitalisation\mat_flow\reblend_v2.xlsx', engine="openpyxl", mode="a", if_sheet_exists="replace") as writer:
df.to_excel(writer, index=False, sheet_name = ft_tags_final[i][j])

I had the same issue and i solved it with using write instead of append. Also i used openpyxl instead of xlsxwriter
from pandas import ExcelWriter
from pandas import ExcelFile
from openpyxl import load_workbook
book = load_workbook('Wallet.xlsx')
writer = pd.ExcelWriter('Wallet.xlsx', engine='openpyxl')
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
#^THIS IS THE MOST IMPORTANT LINES BECAUSE IT GIVES PANDAS THE SHEET
Data.to_excel(writer, sheet_name='Main', header=None, index=False, startcol=number,startrow=counter)

How do I use a loop to write data to multiple Excel sheets in 1 workbook [duplicate]

I want to use excel files to store data elaborated with python. My problem is that I can't add sheets to an existing excel file. Here I suggest a sample code to work with in order to reach this issue
import pandas as pd
import numpy as np
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
x1 = np.random.randn(100, 2)
df1 = pd.DataFrame(x1)
x2 = np.random.randn(100, 2)
df2 = pd.DataFrame(x2)
writer = pd.ExcelWriter(path, engine = 'xlsxwriter')
df1.to_excel(writer, sheet_name = 'x1')
df2.to_excel(writer, sheet_name = 'x2')
writer.save()
writer.close()
This code saves two DataFrames to two sheets, named "x1" and "x2" respectively. If I create two new DataFrames and try to use the same code to add two new sheets, 'x3' and 'x4', the original data is lost.
import pandas as pd
import numpy as np
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
x3 = np.random.randn(100, 2)
df3 = pd.DataFrame(x3)
x4 = np.random.randn(100, 2)
df4 = pd.DataFrame(x4)
writer = pd.ExcelWriter(path, engine = 'xlsxwriter')
df3.to_excel(writer, sheet_name = 'x3')
df4.to_excel(writer, sheet_name = 'x4')
writer.save()
writer.close()
I want an excel file with four sheets: 'x1', 'x2', 'x3', 'x4'.
I know that 'xlsxwriter' is not the only "engine", there is 'openpyxl'. I also saw there are already other people that have written about this issue, but still I can't understand how to do that.
Here a code taken from this link
import pandas
from openpyxl import load_workbook
book = load_workbook('Masterfile.xlsx')
writer = pandas.ExcelWriter('Masterfile.xlsx', engine='openpyxl')
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
data_filtered.to_excel(writer, "Main", cols=['Diff1', 'Diff2'])
writer.save()
They say that it works, but it is hard to figure out how. I don't understand what "ws.title", "ws", and "dict" are in this context.
Which is the best way to save "x1" and "x2", then close the file, open it again and add "x3" and "x4"?

Thank you. I believe that a complete example could be good for anyone else who have the same issue:
import pandas as pd
import numpy as np
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
x1 = np.random.randn(100, 2)
df1 = pd.DataFrame(x1)
x2 = np.random.randn(100, 2)
df2 = pd.DataFrame(x2)
writer = pd.ExcelWriter(path, engine = 'xlsxwriter')
df1.to_excel(writer, sheet_name = 'x1')
df2.to_excel(writer, sheet_name = 'x2')
writer.close()
Here I generate an excel file, from my understanding it does not really matter whether it is generated via the "xslxwriter" or the "openpyxl" engine.
When I want to write without loosing the original data then
import pandas as pd
import numpy as np
from openpyxl import load_workbook
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
book = load_workbook(path)
writer = pd.ExcelWriter(path, engine = 'openpyxl')
writer.book = book
x3 = np.random.randn(100, 2)
df3 = pd.DataFrame(x3)
x4 = np.random.randn(100, 2)
df4 = pd.DataFrame(x4)
df3.to_excel(writer, sheet_name = 'x3')
df4.to_excel(writer, sheet_name = 'x4')
writer.close()
this code do the job!

For creating a new file
x1 = np.random.randn(100, 2)
df1 = pd.DataFrame(x1)
with pd.ExcelWriter('sample.xlsx') as writer:
df1.to_excel(writer, sheet_name='x1')
For appending to the file, use the argument mode='a' in pd.ExcelWriter.
x2 = np.random.randn(100, 2)
df2 = pd.DataFrame(x2)
with pd.ExcelWriter('sample.xlsx', engine='openpyxl', mode='a') as writer:
df2.to_excel(writer, sheet_name='x2')
Default is mode ='w'.
See documentation.

In the example you shared you are loading the existing file into book and setting the writer.book value to be book. In the line writer.sheets = dict((ws.title, ws) for ws in book.worksheets) you are accessing each sheet in the workbook as ws. The sheet title is then ws so you are creating a dictionary of {sheet_titles: sheet} key, value pairs. This dictionary is then set to writer.sheets. Essentially these steps are just loading the existing data from 'Masterfile.xlsx' and populating your writer with them.
Now let's say you already have a file with x1 and x2 as sheets. You can use the example code to load the file and then could do something like this to add x3 and x4.
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
writer = pd.ExcelWriter(path, engine='openpyxl')
df3.to_excel(writer, 'x3', index=False)
df4.to_excel(writer, 'x4', index=False)
writer.save()
That should do what you are looking for.

A simple example for writing multiple data to excel at a time. And also when you want to append data to a sheet on a written excel file (closed excel file).
When it is your first time writing to an excel. (Writing "df1" and "df2" to "1st_sheet" and "2nd_sheet")
import pandas as pd
from openpyxl import load_workbook
df1 = pd.DataFrame([[1],[1]], columns=['a'])
df2 = pd.DataFrame([[2],[2]], columns=['b'])
df3 = pd.DataFrame([[3],[3]], columns=['c'])
excel_dir = "my/excel/dir"
with pd.ExcelWriter(excel_dir, engine='xlsxwriter') as writer:
df1.to_excel(writer, '1st_sheet')
df2.to_excel(writer, '2nd_sheet')
writer.save()
After you close your excel, but you wish to "append" data on the same excel file but another sheet, let's say "df3" to sheet name "3rd_sheet".
book = load_workbook(excel_dir)
with pd.ExcelWriter(excel_dir, engine='openpyxl') as writer:
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
## Your dataframe to append.
df3.to_excel(writer, '3rd_sheet')
writer.save()
Be noted that excel format must not be xls, you may use xlsx one.

Every time you want to save a Pandas DataFrame to an Excel, you may call this function:
import os
def save_excel_sheet(df, filepath, sheetname, index=False):
# Create file if it does not exist
if not os.path.exists(filepath):
df.to_excel(filepath, sheet_name=sheetname, index=index)
# Otherwise, add a sheet. Overwrite if there exists one with the same name.
else:
with pd.ExcelWriter(filepath, engine='openpyxl', if_sheet_exists='replace', mode='a') as writer:
df.to_excel(writer, sheet_name=sheetname, index=index)

I would strongly recommend you work directly with openpyxl since it now supports Pandas DataFrames.
This allows you to concentrate on the relevant Excel and Pandas code.

Can do it without using ExcelWriter, using tools in openpyxl
This can make adding fonts to the new sheet much easier using openpyxl.styles
import pandas as pd
from openpyxl import load_workbook
from openpyxl.utils.dataframe import dataframe_to_rows
#Location of original excel sheet
fileLocation =r'C:\workspace\data.xlsx'
#Location of new file which can be the same as original file
writeLocation=r'C:\workspace\dataNew.xlsx'
data = {'Name':['Tom','Paul','Jeremy'],'Age':[32,43,34],'Salary':[20000,34000,32000]}
#The dataframe you want to add
df = pd.DataFrame(data)
#Load existing sheet as it is
book = load_workbook(fileLocation)
#create a new sheet
sheet = book.create_sheet("Sheet Name")
#Load dataframe into new sheet
for row in dataframe_to_rows(df, index=False, header=True):
sheet.append(row)
#Save the modified excel at desired location
book.save(writeLocation)

You can read existing sheets of your interests, for example, 'x1', 'x2', into memory and 'write' them back prior to adding more new sheets (keep in mind that sheets in a file and sheets in memory are two different things, if you don't read them, they will be lost). This approach uses 'xlsxwriter' only, no openpyxl involved.
import pandas as pd
import numpy as np
path = r"C:\Users\fedel\Desktop\excelData\PhD_data.xlsx"
# begin <== read selected sheets and write them back
df1 = pd.read_excel(path, sheet_name='x1', index_col=0) # or sheet_name=0
df2 = pd.read_excel(path, sheet_name='x2', index_col=0) # or sheet_name=1
writer = pd.ExcelWriter(path, engine='xlsxwriter')
df1.to_excel(writer, sheet_name='x1')
df2.to_excel(writer, sheet_name='x2')
# end ==>
# now create more new sheets
x3 = np.random.randn(100, 2)
df3 = pd.DataFrame(x3)
x4 = np.random.randn(100, 2)
df4 = pd.DataFrame(x4)
df3.to_excel(writer, sheet_name='x3')
df4.to_excel(writer, sheet_name='x4')
writer.save()
writer.close()
If you want to preserve all existing sheets, you can replace above code between begin and end with:
# read all existing sheets and write them back
writer = pd.ExcelWriter(path, engine='xlsxwriter')
xlsx = pd.ExcelFile(path)
for sheet in xlsx.sheet_names:
df = xlsx.parse(sheet_name=sheet, index_col=0)
df.to_excel(writer, sheet_name=sheet)

Another fairly simple way to go about this is to make a method like this:
def _write_frame_to_new_sheet(path_to_file=None, sheet_name='sheet', data_frame=None):
book = None
try:
book = load_workbook(path_to_file)
except Exception:
logging.debug('Creating new workbook at %s', path_to_file)
with pd.ExcelWriter(path_to_file, engine='openpyxl') as writer:
if book is not None:
writer.book = book
data_frame.to_excel(writer, sheet_name, index=False)
The idea here is to load the workbook at path_to_file if it exists and then append the data_frame as a new sheet with sheet_name. If the workbook does not exist, it is created. It seems that neither openpyxl or xlsxwriter append, so as in the example by #Stefano above, you really have to load and then rewrite to append.

#This program is to read from excel workbook to fetch only the URL domain names and write to the existing excel workbook in a different sheet..
#Developer - Nilesh K
import pandas as pd
from openpyxl import load_workbook #for writting to the existing workbook
df = pd.read_excel("urlsearch_test.xlsx")
#You can use the below for the relative path.
# r"C:\Users\xyz\Desktop\Python\
l = [] #To make a list in for loop
#begin
#loop starts here for fetching http from a string and iterate thru the entire sheet. You can have your own logic here.
for index, row in df.iterrows():
try:
str = (row['TEXT']) #string to read and iterate
y = (index)
str_pos = str.index('http') #fetched the index position for http
str_pos1 = str.index('/', str.index('/')+2) #fetched the second 3rd position of / starting from http
str_op = str[str_pos:str_pos1] #Substring the domain name
l.append(str_op) #append the list with domain names
#Error handling to skip the error rows and continue.
except ValueError:
print('Error!')
print(l)
l = list(dict.fromkeys(l)) #Keep distinct values, you can comment this line to get all the values
df1 = pd.DataFrame(l,columns=['URL']) #Create dataframe using the list
#end
#Write using openpyxl so it can be written to same workbook
book = load_workbook('urlsearch_test.xlsx')
writer = pd.ExcelWriter('urlsearch_test.xlsx',engine = 'openpyxl')
writer.book = book
df1.to_excel(writer,sheet_name = 'Sheet3')
writer.save()
writer.close()
#The below can be used to write to a different workbook without using openpyxl
#df1.to_excel(r"C:\Users\xyz\Desktop\Python\urlsearch1_test.xlsx",index='false',sheet_name='sheet1')

if you want to add empty sheet
xw = pd.ExcelWriter(file_path, engine='xlsxwriter')
pd.DataFrame().to_excel(xw, 'sheet11')
if you get empty sheet
sheet = xw.sheets['sheet11']

import pandas as pd
import openpyxl
writer = pd.ExcelWriter('test.xlsx', engine='openpyxl')
data_df.to_excel(writer, 'sheet_name')
writer.save()
writer.close()

The following solution worked for me:
# dataframe to save
df = pd.DataFrame({"A":[1,2], "B":[3,4]})
# path where you want to save
path = "./..../..../.../test.xlsx"
# if an excel sheet named `test` is already present append on sheet 2
if os.path.isfile(path):
with pd.ExcelWriter(path, mode='a') as writer:
df.to_excel(writer, sheet_name= "sheet_2")
else:
# if not present then write to a excel file on sheet 1
with pd.ExcelWriter(path) as writer:
df.to_excel(writer, sheet_name= "sheet_1")
Now, if you want to write multiple dataframes on different sheets, simply add a loop and keep on changing the sheet_name.

Loop update existing excel template

Trying to write a script where I currently have an excel VBA sheet that has two tabs with 1st being a graph and second being a backend file. Backend is updated by a master file. In the master file there is a city column where I want to loop through all the unique city rows write those rows in to the VBA file and save the VBA file with the city's name.
master_backend = pd.read_excel(path)
city = master_backend[(master_backend["City"]=="NY")]
def append_df_to_excel(filename, df, sheet_name='Sheet1', startrow=None,
truncate_sheet=False,
**to_excel_kwargs):
from openpyxl import load_workbook
import pandas as pd
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
writer = pd.ExcelWriter(filename, engine='openpyxl')
try:
FileNotFoundError
except NameError:
FileNotFoundError = IOError
try:
writer.book = load_workbook(filename, keep_vba = True)
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
if truncate_sheet and sheet_name in writer.book.sheetnames:
idx = writer.book.sheetnames.index(sheet_name)
writer.book.remove(writer.book.worksheets[idx])
writer.book.create_sheet(sheet_name, idx)
writer.sheets = {ws.title:ws for ws in writer.book.worksheets}
except FileNotFoundError:
pass
if startrow is None:
startrow = 0
df.to_excel(writer, sheet_name, startrow=startrow, **to_excel_kwargs)
writer.save()
Essentially what I want is 5 files since there are 5 cities all named with their city name

as I don't know VBA and you posted this under the python tag I'll provide my take on this.
assuming your datasheet is called file you could try something like this :
import shutil
for city in master_backend.City.unique():
df = master_backend.loc[master_backend.City == city]
shutil.copy(file,f"{city}.xlsx")
append_df_excel(f"{city}.xlsx", df,sheet_name='Backend')
cracking function btw, I would use put some doc strings in it for easy of use : )

I think you can simplify this script significantly by understanding that pandas will create a dataframe for you when you read the excel file. Then it's just a simple matter of collecting the info you want from the dataframe and re-writting it to a file. It's unclear what you want in your new file, but suppose you just want to filter the second sheet and keep everything in the first sheet it might look like this.
# Open the file,
# NOTE: when you open the file, if there are multiple sheets
# then the result is a dictionary of dataframes keyed on the sheet name
master_data = pd.read_excel(file_path, ....)
# Assuming second sheet name is 'City'
city_df=master_data['City']
# Replace 'columnName' with the name of the column (if includes headers) or column number
for city in pd.unique(city_df['columnName']):
with pd.ExcelWriter(city + '.xlsx') as writer:
master_data['Sheet1'].to_excel(writer, sheet_name='Sheet1')
city_df[city_df['columnName']==city].to_excel(writer, sheet_name='City')

Append existing excel sheet with new dataframe using python pandas

I currently have this code. It works perfectly.
It loops through excel files in a folder,
removes the first 2 rows, then saves them as individual excel files,
and it also saves the files in the loop as an appended file.
Currently the appended file overwrites the existing file each time I run the code.
I need to append the new data to the bottom of the already existing excel sheet ('master_data.xlsx)
dfList = []
path = 'C:\\Test\\TestRawFile'
newpath = 'C:\\Path\\To\\New\\Folder'
for fn in os.listdir(path):
# Absolute file path
file = os.path.join(path, fn)
if os.path.isfile(file):
# Import the excel file and call it xlsx_file
xlsx_file = pd.ExcelFile(file)
# View the excel files sheet names
xlsx_file.sheet_names
# Load the xlsx files Data sheet as a dataframe
df = xlsx_file.parse('Sheet1',header= None)
df_NoHeader = df[2:]
data = df_NoHeader
# Save individual dataframe
data.to_excel(os.path.join(newpath, fn))
dfList.append(data)
appended_data = pd.concat(dfList)
appended_data.to_excel(os.path.join(newpath, 'master_data.xlsx'))
I thought this would be a simple task, but I guess not.
I think I need to bring in the master_data.xlsx file as a dataframe, then match the index up with the new appended data, and save it back out. Or maybe there is an easier way. Any Help is appreciated.

UPDATE [2022-01-08]: starting from version 1.4.0 Pandas supports appending to existing Excel sheet, preserving the old contents, "out of the box"!
Good job Pandas Team!
Excerpt from the ExcelWriter documentation:
if_sheet_exists : {'error', 'new', 'replace', 'overlay'}, default 'error'
How to behave when trying to write to a sheet that already
exists (append mode only).
...
* overlay: Write contents to the existing sheet without removing the old contents.
.. versionadded:: 1.3.0
.. versionchanged:: 1.4.0
Added ``overlay`` option
For Pandas versions < 1.4.0 please find below a helper function for appending a Pandas DataFrame to an existing Excel file.
If an Excel file doesn't exist then it will be created.
UPDATE [2021-09-12]: fixed for Pandas 1.3.0+
The following functions have been tested with:
Pandas 1.3.2
OpenPyxl 3.0.7
from pathlib import Path
from copy import copy
from typing import Union, Optional
import numpy as np
import pandas as pd
import openpyxl
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
def copy_excel_cell_range(
src_ws: openpyxl.worksheet.worksheet.Worksheet,
min_row: int = None,
max_row: int = None,
min_col: int = None,
max_col: int = None,
tgt_ws: openpyxl.worksheet.worksheet.Worksheet = None,
tgt_min_row: int = 1,
tgt_min_col: int = 1,
with_style: bool = True
) -> openpyxl.worksheet.worksheet.Worksheet:
"""
copies all cells from the source worksheet [src_ws] starting from [min_row] row
and [min_col] column up to [max_row] row and [max_col] column
to target worksheet [tgt_ws] starting from [tgt_min_row] row
and [tgt_min_col] column.
#param src_ws: source worksheet
#param min_row: smallest row index in the source worksheet (1-based index)
#param max_row: largest row index in the source worksheet (1-based index)
#param min_col: smallest column index in the source worksheet (1-based index)
#param max_col: largest column index in the source worksheet (1-based index)
#param tgt_ws: target worksheet.
If None, then the copy will be done to the same (source) worksheet.
#param tgt_min_row: target row index (1-based index)
#param tgt_min_col: target column index (1-based index)
#param with_style: whether to copy cell style. Default: True
#return: target worksheet object
"""
if tgt_ws is None:
tgt_ws = src_ws
# https://stackoverflow.com/a/34838233/5741205
for row in src_ws.iter_rows(min_row=min_row, max_row=max_row,
min_col=min_col, max_col=max_col):
for cell in row:
tgt_cell = tgt_ws.cell(
row=cell.row + tgt_min_row - 1,
column=cell.col_idx + tgt_min_col - 1,
value=cell.value
)
if with_style and cell.has_style:
# tgt_cell._style = copy(cell._style)
tgt_cell.font = copy(cell.font)
tgt_cell.border = copy(cell.border)
tgt_cell.fill = copy(cell.fill)
tgt_cell.number_format = copy(cell.number_format)
tgt_cell.protection = copy(cell.protection)
tgt_cell.alignment = copy(cell.alignment)
return tgt_ws
def append_df_to_excel(
filename: Union[str, Path],
df: pd.DataFrame,
sheet_name: str = 'Sheet1',
startrow: Optional[int] = None,
max_col_width: int = 30,
autofilter: bool = False,
fmt_int: str = "#,##0",
fmt_float: str = "#,##0.00",
fmt_date: str = "yyyy-mm-dd",
fmt_datetime: str = "yyyy-mm-dd hh:mm",
truncate_sheet: bool = False,
storage_options: Optional[dict] = None,
**to_excel_kwargs
) -> None:
"""
Append a DataFrame [df] to existing Excel file [filename]
into [sheet_name] Sheet.
If [filename] doesn't exist, then this function will create it.
#param filename: File path or existing ExcelWriter
(Example: '/path/to/file.xlsx')
#param df: DataFrame to save to workbook
#param sheet_name: Name of sheet which will contain DataFrame.
(default: 'Sheet1')
#param startrow: upper left cell row to dump data frame.
Per default (startrow=None) calculate the last row
in the existing DF and write to the next row...
#param max_col_width: maximum column width in Excel. Default: 40
#param autofilter: boolean - whether add Excel autofilter or not. Default: False
#param fmt_int: Excel format for integer numbers
#param fmt_float: Excel format for float numbers
#param fmt_date: Excel format for dates
#param fmt_datetime: Excel format for datetime's
#param truncate_sheet: truncate (remove and recreate) [sheet_name]
before writing DataFrame to Excel file
#param storage_options: dict, optional
Extra options that make sense for a particular storage connection, e.g. host, port,
username, password, etc., if using a URL that will be parsed by fsspec, e.g.,
starting “s3://”, “gcs://”.
#param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()`
[can be a dictionary]
#return: None
Usage examples:
>>> append_df_to_excel('/tmp/test.xlsx', df, autofilter=True,
freeze_panes=(1,0))
>>> append_df_to_excel('/tmp/test.xlsx', df, header=None, index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False, startrow=25)
>>> append_df_to_excel('/tmp/test.xlsx', df, index=False,
fmt_datetime="dd.mm.yyyy hh:mm")
(c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile)
"""
def set_column_format(ws, column_letter, fmt):
for cell in ws[column_letter]:
cell.number_format = fmt
filename = Path(filename)
file_exists = filename.is_file()
# process parameters
# calculate first column number
# if the DF will be written using `index=True`, then `first_col = 2`, else `first_col = 1`
first_col = int(to_excel_kwargs.get("index", True)) + 1
# ignore [engine] parameter if it was passed
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
# save content of existing sheets
if file_exists:
wb = load_workbook(filename)
sheet_names = wb.sheetnames
sheet_exists = sheet_name in sheet_names
sheets = {ws.title: ws for ws in wb.worksheets}
with pd.ExcelWriter(
filename.with_suffix(".xlsx"),
engine="openpyxl",
mode="a" if file_exists else "w",
if_sheet_exists="new" if file_exists else None,
date_format=fmt_date,
datetime_format=fmt_datetime,
storage_options=storage_options
) as writer:
if file_exists:
# try to open an existing workbook
writer.book = wb
# get the last row in the existing Excel sheet
# if it was not specified explicitly
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
# truncate sheet
if truncate_sheet and sheet_name in writer.book.sheetnames:
# index of [sheet_name] sheet
idx = writer.book.sheetnames.index(sheet_name)
# remove [sheet_name]
writer.book.remove(writer.book.worksheets[idx])
# create an empty sheet [sheet_name] using old index
writer.book.create_sheet(sheet_name, idx)
# copy existing sheets
writer.sheets = sheets
else:
# file doesn't exist, we are creating a new one
startrow = 0
# write out the DataFrame to an ExcelWriter
df.to_excel(writer, sheet_name=sheet_name, **to_excel_kwargs)
worksheet = writer.sheets[sheet_name]
if autofilter:
worksheet.auto_filter.ref = worksheet.dimensions
for xl_col_no, dtyp in enumerate(df.dtypes, first_col):
col_no = xl_col_no - first_col
width = max(df.iloc[:, col_no].astype(str).str.len().max(),
len(df.columns[col_no]) + 6)
width = min(max_col_width, width)
column_letter = get_column_letter(xl_col_no)
worksheet.column_dimensions[column_letter].width = width
if np.issubdtype(dtyp, np.integer):
set_column_format(worksheet, column_letter, fmt_int)
if np.issubdtype(dtyp, np.floating):
set_column_format(worksheet, column_letter, fmt_float)
if file_exists and sheet_exists:
# move (append) rows from new worksheet to the `sheet_name` worksheet
wb = load_workbook(filename)
# retrieve generated worksheet name
new_sheet_name = set(wb.sheetnames) - set(sheet_names)
if new_sheet_name:
new_sheet_name = list(new_sheet_name)[0]
# copy rows written by `df.to_excel(...)` to
copy_excel_cell_range(
src_ws=wb[new_sheet_name],
tgt_ws=wb[sheet_name],
tgt_min_row=startrow + 1,
with_style=True
)
# remove new (generated by Pandas) worksheet
del wb[new_sheet_name]
wb.save(filename)
wb.close()
Old version (tested with Pandas 1.2.3 and Openpyxl 3.0.5):
import os
from openpyxl import load_workbook
def append_df_to_excel(filename, df, sheet_name='Sheet1', startrow=None,
truncate_sheet=False,
**to_excel_kwargs):
"""
Append a DataFrame [df] to existing Excel file [filename]
into [sheet_name] Sheet.
If [filename] doesn't exist, then this function will create it.
#param filename: File path or existing ExcelWriter
(Example: '/path/to/file.xlsx')
#param df: DataFrame to save to workbook
#param sheet_name: Name of sheet which will contain DataFrame.
(default: 'Sheet1')
#param startrow: upper left cell row to dump data frame.
Per default (startrow=None) calculate the last row
in the existing DF and write to the next row...
#param truncate_sheet: truncate (remove and recreate) [sheet_name]
before writing DataFrame to Excel file
#param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()`
[can be a dictionary]
#return: None
Usage examples:
>>> append_df_to_excel('d:/temp/test.xlsx', df)
>>> append_df_to_excel('d:/temp/test.xlsx', df, header=None, index=False)
>>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2',
index=False)
>>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2',
index=False, startrow=25)
(c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile)
"""
# Excel file doesn't exist - saving and exiting
if not os.path.isfile(filename):
df.to_excel(
filename,
sheet_name=sheet_name,
startrow=startrow if startrow is not None else 0,
**to_excel_kwargs)
return
# ignore [engine] parameter if it was passed
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
writer = pd.ExcelWriter(filename, engine='openpyxl', mode='a')
# try to open an existing workbook
writer.book = load_workbook(filename)
# get the last row in the existing Excel sheet
# if it was not specified explicitly
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
# truncate sheet
if truncate_sheet and sheet_name in writer.book.sheetnames:
# index of [sheet_name] sheet
idx = writer.book.sheetnames.index(sheet_name)
# remove [sheet_name]
writer.book.remove(writer.book.worksheets[idx])
# create an empty sheet [sheet_name] using old index
writer.book.create_sheet(sheet_name, idx)
# copy existing sheets
writer.sheets = {ws.title:ws for ws in writer.book.worksheets}
if startrow is None:
startrow = 0
# write out the new sheet
df.to_excel(writer, sheet_name, startrow=startrow, **to_excel_kwargs)
# save the workbook
writer.save()
Usage examples:
filename = r'C:\OCC.xlsx'
append_df_to_excel(filename, df)
append_df_to_excel(filename, df, header=None, index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False, startrow=25)
c:/temp/test.xlsx:
PS you may also want to specify header=None if you don't want to duplicate column names...
UPDATE: you may also want to check this old solution

If you aren't strictly looking for an excel file, then get the output as csv file and just copy the csv to a new excel file.
Note: this only works when you have less than 1000 columns since csv has a limit on the number of columns you can write.
df.to_csv('filepath', mode='a', index = False, header=None)
mode='a' means append.
This is a roundabout way it but works neat!

Building on MaxU and others' code and comments but simplifying to only fix the bug with pandas ExcelWriter that causes to_excel to create a new sheet rather than append to an existing sheet in append mode.
As others have noted, to_excel uses the ExcelWriter.sheets property and this is not populated when by ExcelWriter.
Fix is a one liner, otherwise code is standard pandas approach as documented in to_excel.
# xl_path is destination xlsx spreadsheet
with pd.ExcelWriter(xl_path, 'openpyxl', mode='a') as writer:
# fix line
writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets)
df.to_excel(writer, sheet_name)

This worked for me
import os
import openpyxl
import pandas as pd
from openpyxl.utils.dataframe import dataframe_to_rows
file = r"myfile.xlsx"
df = pd.DataFrame({'A': 1, 'B': 2})
# create excel file
if os.path.isfile(file): # if file already exists append to existing file
workbook = openpyxl.load_workbook(file) # load workbook if already exists
sheet = workbook['my_sheet_name'] # declare the active sheet
# append the dataframe results to the current excel file
for row in dataframe_to_rows(df, header = False, index = False):
sheet.append(row)
workbook.save(file) # save workbook
workbook.close() # close workbook
else: # create the excel file if doesn't already exist
with pd.ExcelWriter(path = file, engine = 'openpyxl') as writer:
df.to_excel(writer, index = False, sheet_name = 'my_sheet_name')

import pandas as pd
import openpyxl
workbook = openpyxl.load_workbook("test.xlsx")
writer = pd.ExcelWriter('test.xlsx', engine='openpyxl')
writer.book = workbook
writer.sheets = dict((ws.title, ws) for ws in workbook.worksheets)
data_df.to_excel(writer, 'Existing_sheetname')
writer.save()
writer.close()

If you use ExcelWriter on the sheet every time it is going to override the previous sheet and all that will be visible is the last data sheet you appended to the workbook.
Instead you can maintain a counter that is 1 initially for which you need to initialize the excel sheet and add initial data using the existing approach of
writer = pd.ExcelWriter(output_file, engine='openpyxl')
df = pd.read_excel(output_file, sheet_name='TestSheet1')
or you can use the following approach i used. to load the workbook next time you want to use it or else file not find exception if you try to load it in the first case.
USage:
from bs4 import BeautifulSoup
import requests
import pandas as pd
from openpyxl import load_workbook
urls = ["http://millenniumcricketleague.com/Home/ShowTeam.aspx?tid=22",
"http://millenniumcricketleague.com/Home/ShowTeam.aspx?tid=40"]
path = "F:\meta_1.xlsx"
writer = pd.ExcelWriter(path,engine='openpyxl')
counter = 1
for url in urls:
table_data = []
final = []
html_content = requests.get(url).text
soup = BeautifulSoup(html_content, "lxml")
x = soup.find_all('table')
for table in x[1:]:
for tr in table.find_all("tr"):
newrow = []
for td in tr.find_all("td"):
newrow.append(td.text.replace('\n', ' ').strip())
table_data.append(newrow)
df = pd.DataFrame(table_data)
sheetname = 'Sheet%s' % counter
if(counter!=1):
writer.book = load_workbook(path)
df.to_excel(writer, sheet_name=sheetname)
counter = counter + 1
writer.save()
NO need to close the excelwriter. its an automatic function. Will show you a warning if you define it explicitly

This question has been out here a while. The answer is ok, but I believe this will solve most peoples question.
simply use glob to access the files in a specific directory, loop through them, create a dataframe of each file, append it to the last one, then export to a folder. I also included commented out code to run through this with csvs.
import os
import pandas as pd
import glob
# put in path to folder with files you want to append
# *.xlsx or *.csv will get all files of that type
path = "C:/Users/Name/Folder/*.xlsx"
#path = "C:/Users/Name/Folder/*.csv"
# initialize a empty df
appended_data = pd.DataFrame()
#loop through each file in the path
for file in glob.glob(path):
print(file)
# create a df of that file path
df = pd.read_excel(file, sheet_name = 0)
#df = pd.read_csv(file, sep=',')
# appened it
appended_data = appended_data.append(df)
appended_data
# export the appeneded data to a folder of your choice
exportPath = 'C:/My/EXPORT/PATH/appended_dataExport.csv'
appended_data.to_csv(os.path.join(exportPath),index=False)

Complementing to #david, if you dont care the index and you can use .csv, this function helps to append any df to an existing csv
def append_df(self, path_file, df):
with open(path_file, 'a+') as f:
df.to_csv(f, header=f.tell() == 0, encoding='utf-8', index=False)
Notes:
a+ create the file if it doesnot exist
f.tell() == 0 add header if the first row

from openpyxl import load_workbook
wb = load_workbook(filepath)
ws = wb["Sheet1"]
df = dataframe.values.tolist()
for i in range(len(df)):
ws.append(df[i])
wb.save(filepath)

Append DataFrame to existing excel file
Use ExcelWriter to append DataFrame to an existing excel file. This is a simple approach and uses the existing library features.
with pd.ExcelWriter('existing_excel_file.xlsx',mode='a') as writer:
df.to_excel(writer, sheet_name='existing_sheet_name')
For detailed examples refer to pandas read Excel File with Examples